diff --git a/ArtificialAutism/src/main/java/DataLayer/DataMapper.java b/ArtificialAutism/src/main/java/DataLayer/DataMapper.java index 43710e29..4671ca99 100644 --- a/ArtificialAutism/src/main/java/DataLayer/DataMapper.java +++ b/ArtificialAutism/src/main/java/DataLayer/DataMapper.java @@ -5,69 +5,39 @@ */ package DataLayer; -import FunctionLayer.SimilarityMatrix; -import FunctionLayer.CustomError; -import com.google.common.collect.MapMaker; import java.sql.Connection; import java.sql.PreparedStatement; import java.sql.ResultSet; import java.sql.SQLException; -import java.sql.Statement; -import java.util.ArrayList; -import java.util.HashMap; -import java.util.LinkedHashMap; -import java.util.List; -import java.util.Map; -import java.util.concurrent.ConcurrentMap; +import java.util.*; import java.util.logging.Level; import java.util.logging.Logger; /** - * * @author install1 */ public class DataMapper { - public static void createTables() throws CustomError { - Connection l_cCon = null; - PreparedStatement l_pStatement = null; - ResultSet l_rsSearch = null; - try { - l_cCon = DBCPDataSource.getConnection(); - String l_sSQL = "CREATE TABLE IF NOT EXISTS `ArtificialAutism`.`Sentences` (`Strings` text NOT NULL)"; - l_pStatement = l_cCon.prepareStatement(l_sSQL); - l_pStatement.execute(); - } catch (SQLException ex) { - throw new CustomError("failed in DataMapper " + ex.getMessage()); - } finally { - CloseConnections(l_pStatement, l_rsSearch, l_cCon); - } - } - - public static ConcurrentMap getAllStrings() throws CustomError { - ConcurrentMap allStrings = new MapMaker().concurrencyLevel(2).makeMap(); + public static ArrayList getAllStrings() throws SQLException { Connection l_cCon = null; PreparedStatement l_pStatement = null; ResultSet l_rsSearch = null; + ArrayList arrayListStr = new ArrayList(); try { l_cCon = DBCPDataSource.getConnection(); String l_sSQL = "SELECT * FROM `Sentences`"; l_pStatement = l_cCon.prepareStatement(l_sSQL); l_rsSearch = l_pStatement.executeQuery(); - int ij = 0; while (l_rsSearch.next()) { - allStrings.put(ij, l_rsSearch.getString(1)); - ij++; + arrayListStr.add(l_rsSearch.getString(1)); } - } catch (SQLException ex) { - throw new CustomError("failed in DataMapper " + ex.getMessage()); } finally { CloseConnections(l_pStatement, l_rsSearch, l_cCon); } - return allStrings; + return arrayListStr; } - public static void InsertMYSQLStrings(ConcurrentMap str) throws CustomError { + public static void InsertMYSQLStrings(ArrayList str) throws SQLException { Connection l_cCon = null; PreparedStatement l_pStatement = null; ResultSet l_rsSearch = null; @@ -75,14 +45,10 @@ public class DataMapper { try { l_cCon = DBCPDataSource.getConnection(); l_pStatement = l_cCon.prepareStatement(l_sSQL); - for (String str1 : str.values()) { - //System.out.println("adding str1: " + str1 + "\n"); + for (String str1 : str) { l_pStatement.setString(1, str1); - l_pStatement.addBatch(); + l_pStatement.execute(); } - l_pStatement.executeBatch(); - } catch (SQLException ex) { - throw new CustomError("failed in DataMapper " + ex.getMessage()); } finally { CloseConnections(l_pStatement, l_rsSearch, l_cCon); } diff --git a/ArtificialAutism/src/main/java/FunctionLayer/CustomError.java b/ArtificialAutism/src/main/java/FunctionLayer/CustomError.java deleted file mode 100644 index a7988d55..00000000 --- a/ArtificialAutism/src/main/java/FunctionLayer/CustomError.java +++ /dev/null @@ -1,17 +0,0 @@ -/* - * To change this license header, choose License Headers in Project Properties. - * To change this template file, choose Tools | Templates - * and open the template in the editor. - */ -package FunctionLayer; - -/** - * - * @author install1 - */ -public class CustomError extends Exception { - - public CustomError(String msg) { - super(msg); - } -} diff --git a/ArtificialAutism/src/main/java/FunctionLayer/Datahandler.kt b/ArtificialAutism/src/main/java/FunctionLayer/Datahandler.kt index 6d64eaa8..7ca1bd14 100644 --- a/ArtificialAutism/src/main/java/FunctionLayer/Datahandler.kt +++ b/ArtificialAutism/src/main/java/FunctionLayer/Datahandler.kt @@ -7,32 +7,29 @@ package FunctionLayer import DataLayer.DataMapper import FunctionLayer.StanfordParser.SentimentAnalyzerTest -import FunctionLayer.StanfordParser.SentimentValueCache import com.google.common.base.Stopwatch -import com.google.common.collect.MapMaker import edu.stanford.nlp.ie.AbstractSequenceClassifier import edu.stanford.nlp.ie.crf.CRFClassifier +import edu.stanford.nlp.ling.CoreAnnotations import edu.stanford.nlp.ling.CoreLabel +import edu.stanford.nlp.ling.TaggedWord import edu.stanford.nlp.parser.lexparser.LexicalizedParser import edu.stanford.nlp.pipeline.Annotation import edu.stanford.nlp.pipeline.CoreDocument import edu.stanford.nlp.pipeline.StanfordCoreNLP import edu.stanford.nlp.tagger.maxent.MaxentTagger import edu.stanford.nlp.trees.GrammaticalStructureFactory -import edu.stanford.nlp.trees.TreebankLanguagePack -import kotlinx.coroutines.* -import java.io.IOException -import java.io.UnsupportedEncodingException -import java.net.* -import java.sql.SQLException +import edu.stanford.nlp.trees.Tree +import edu.stanford.nlp.util.CoreMap +import kotlinx.coroutines.Dispatchers +import kotlinx.coroutines.launch +import kotlinx.coroutines.runBlocking +import kotlinx.coroutines.yield import java.util.* -import java.util.concurrent.ConcurrentMap -import java.util.concurrent.CountDownLatch import java.util.concurrent.TimeUnit -import java.util.function.Consumer -import java.util.logging.Level -import java.util.logging.Logger +import java.util.regex.Pattern import kotlin.collections.ArrayList +import kotlin.collections.HashMap /** @@ -41,338 +38,266 @@ import kotlin.collections.ArrayList */ public class Datahandler { private val stopwatch: Stopwatch - fun shiftReduceParserInitiate() = runBlocking { - val cdl = CountDownLatch(2) - coroutineScope { - val job = launch(Dispatchers.Default) { - propsSentiment.setProperty("parse.model", lexParserEnglishRNN) - propsSentiment.setProperty("sentiment.model", sentimentModel) - propsSentiment.setProperty("parse.maxlen", "90") - propsSentiment.setProperty("threads", "5") - propsSentiment.setProperty("pos.maxlen", "90") - propsSentiment.setProperty("tokenize.maxlen", "90") - propsSentiment.setProperty("ssplit.maxlen", "90") - propsSentiment.setProperty("annotators", "tokenize,ssplit,pos,parse,sentiment,lemma,stopword") //coref too expensive memorywise - propsSentiment.setProperty("customAnnotatorClass.stopword", "FunctionLayer.StopwordAnnotator") - propsSentiment.setProperty(StopwordAnnotator.STOPWORDS_LIST, customStopWordList) - propsSentiment.setProperty("tokenize.options", "untokenizable=firstKeep") - pipelineSentiment = StanfordCoreNLP(propsSentiment) - tagger = MaxentTagger(taggerPath) - lp = LexicalizedParser.loadModel(lexParserEnglishRNN, *options) - tlp = lp.getOp().langpack() - gsf = tlp.grammaticalStructureFactory() - cdl.countDown() - yield() - } - job.join() - } - coroutineScope { - val job = launch(Dispatchers.Default) { - try { - classifier = CRFClassifier.getClassifierNoExceptions(nerModel) - } catch (ex: ClassCastException) { - Logger.getLogger(Datahandler::class.java.name).log(Level.SEVERE, null, ex) - } - cdl.countDown() - yield() - } - job.join() - } - try { - cdl.await() - } catch (ex: InterruptedException) { - //System.out.println("cdl await interrupted: " + ex.getLocalizedMessage() + "\n"); - } + private val EXPIRE_TIME_IN_MINUTES = TimeUnit.MINUTES.convert(30, TimeUnit.MINUTES) + private var pipelineAnnotationCache: HashMap + private var pipelineSentimentAnnotationCache = HashMap() + private var coreDocumentAnnotationCache: HashMap + private var jmweAnnotationCache = HashMap() + private val stringCache = ArrayList() + private val nerModel = "edu/stanford/nlp/models/ner/english.all.3class.caseless.distsim.crf.ser.gz" + private var tagger: MaxentTagger = MaxentTagger() + private var gsf: GrammaticalStructureFactory + private var classifier: AbstractSequenceClassifier + + //SentimentAnalyzer Hashmaps + private var tokenizeCountingHashMap: HashMap = HashMap() + private var taggedWordListHashMap: HashMap>> = HashMap() + private var retrieveTGWListHashMap: HashMap> = + HashMap() + private var sentences1HashMap: HashMap> = HashMap() + private var sentencesSentimentHashMap: HashMap> = HashMap() + private var trees1HashMap: HashMap> = HashMap() + + constructor() { + stopwatch = Stopwatch.createUnstarted() + jmweAnnotationCache = HashMap() + pipelineAnnotationCache = HashMap() + pipelineSentimentAnnotationCache = HashMap() + coreDocumentAnnotationCache = HashMap() + gsf = initiateGrammaticalStructureFactory() + classifier = CRFClassifier.getClassifierNoExceptions(nerModel) + } + + fun initiateGrammaticalStructureFactory(): GrammaticalStructureFactory { + val options = arrayOf("-maxLength", "100") + val lexParserEnglishRNN = "edu/stanford/nlp/models/lexparser/englishRNN.ser.gz" + val lp = LexicalizedParser.loadModel(lexParserEnglishRNN, *options) + val tlp = lp.getOp().langpack() + return tlp.grammaticalStructureFactory() + } + + public fun pipeLineSetUp(): StanfordCoreNLP { + val props = Properties() + val shiftReduceParserPath = "edu/stanford/nlp/models/srparser/englishSR.ser.gz" + val nerModel2 = "edu/stanford/nlp/models/ner/english.conll.4class.caseless.distsim.crf.ser.gz" + val nerModel3 = "edu/stanford/nlp/models/ner/english.muc.7class.caseless.distsim.crf.ser.gz" + props.setProperty("annotators", "tokenize,ssplit,pos,lemma,ner,parse") + props.setProperty("parse.model", shiftReduceParserPath) + props.setProperty("parse.maxlen", "90") + props.setProperty("parse.binaryTrees", "true") + props.setProperty("threads", "5") + props.setProperty("pos.maxlen", "90") + props.setProperty("tokenize.maxlen", "90") + props.setProperty("ssplit.maxlen", "90") + props.setProperty("lemma.maxlen", "90") + props.setProperty("ner.model", "$nerModel,$nerModel2,$nerModel3") + props.setProperty("ner.combinationMode", "HIGH_RECALL") + props.setProperty("regexner.ignorecase", "true") + props.setProperty("ner.fine.regexner.ignorecase", "true") + props.setProperty("tokenize.options", "untokenizable=firstKeep") + return StanfordCoreNLP(props) + } + + fun shiftReduceParserInitiate(): StanfordCoreNLP { + val propsSentiment = Properties() + val lexParserEnglishRNN = "edu/stanford/nlp/models/lexparser/englishRNN.ser.gz" + val sentimentModel = "edu/stanford/nlp/models/sentiment/sentiment.ser.gz" + val taggerPath = "edu/stanford/nlp/models/pos-tagger/english-left3words/english-left3words-distsim.tagger" + val customStopWordList = "start,starts,period,periods,a,an,and,are,as,at,be,but,by,for,if,in,into,is,it,no,not,of," + + "on,or,such,that,the,their,then,there,these,they,this,to,was,will,with" + propsSentiment.setProperty("parse.model", lexParserEnglishRNN) + propsSentiment.setProperty("sentiment.model", sentimentModel) + propsSentiment.setProperty("parse.maxlen", "90") + propsSentiment.setProperty("threads", "5") + propsSentiment.setProperty("pos.maxlen", "90") + propsSentiment.setProperty("tokenize.maxlen", "90") + propsSentiment.setProperty("ssplit.maxlen", "90") + propsSentiment.setProperty("annotators", "tokenize,ssplit,pos,parse,sentiment,lemma,stopword") //coref too expensive memorywise + propsSentiment.setProperty("customAnnotatorClass.stopword", "FunctionLayer.StopwordAnnotator") + propsSentiment.setProperty(StopwordAnnotator.STOPWORDS_LIST, customStopWordList) + propsSentiment.setProperty("tokenize.options", "untokenizable=firstKeep") + tagger = MaxentTagger(taggerPath) + println("finished shiftReduceParserInitiate\n") + return StanfordCoreNLP(propsSentiment) } fun updateStringCache() { - try { - checkIfUpdateStrings() - } catch (ex: CustomError) { - Logger.getLogger(Datahandler::class.java.name).log(Level.SEVERE, null, ex) - } - } - - @get:Throws(SQLException::class, IOException::class, CustomError::class) - private val cache: Map - private get() = DataMapper.getAllStrings() - - @Throws(SQLException::class, IOException::class) - fun initiateMYSQL() { - try { - DataMapper.createTables() - stringCache.putAll(cache) - } catch (ex: CustomError) { - Logger.getLogger(Datahandler::class.java - .name).log(Level.SEVERE, null, ex) - } - } - - fun instantiateAnnotationMapJMWE() { - if (!stringCache.isEmpty()) { - val jmweAnnotation = PipelineJMWESingleton.INSTANCE.getJMWEAnnotation(stringCache.values) - for ((key, value) in jmweAnnotation) { - jmweAnnotationCache[key] = value - } - } - } - - fun instantiateAnnotationMap() = runBlocking { - if (!stringCache.isEmpty()) { - val Annotationspipeline = MapMaker().concurrencyLevel(5).makeMap() - val AnnotationspipelineSentiment = MapMaker().concurrencyLevel(5).makeMap() - coroutineScope { - for (str in stringCache.values) { - val job = launch(Dispatchers.Default) { - val strAnno = Annotation(str) - strAnno.compact() - Annotationspipeline[str] = strAnno - val strAnno2 = Annotation(str) - strAnno2.compact() - AnnotationspipelineSentiment[str] = strAnno2 - yield() - } - job.join(); - } - } - System.out.println("PRE getMultipleCoreDocumentsWaySuggestion lag") - val coreDocumentpipelineMap = getMultipleCoreDocumentsWaySuggestion(stringCache.values, pipeline) - //System.out.println("post getMultipleCoreDocumentsWaySuggestion instantiateAnnotationMap lag") - pipeline.annotate(Annotationspipeline.values, 4) - pipelineSentiment!!.annotate(AnnotationspipelineSentiment.values, 4) - //System.out.println("reached second job instantiateAnnotationMap lag"); - coroutineScope { - for (i in Annotationspipeline.entries) { - val job = launch(Dispatchers.Default) { - i.value.compact() - pipelineAnnotationCache[i.key] = i.value - yield() - } - job.join(); - } - - for (i in AnnotationspipelineSentiment.entries) { - val job = launch(Dispatchers.Default) { - i.value.compact() - pipelineSentimentAnnotationCache[i.key] = i.value - yield() - } - job.join(); - } - } - System.out.println("post Annotationspipeline lag") - for (i in coreDocumentpipelineMap.entries) { - coreDocumentAnnotationCache[i.key] = i.value - } - - } - } - - private fun futuresReturnOverallEvaluation(similarityMatrixes: List): ConcurrentMap { - var strmapreturn = MapMaker().concurrencyLevel(6).makeMap() - if (!similarityMatrixes.isEmpty()) { - for (SMX in similarityMatrixes) { - strmapreturn = addSMXToMapReturn(strmapreturn, SMX) - } - } - return strmapreturn - } - - private fun addSMXToMapReturn(strmapreturn: ConcurrentMap, SMX: SimilarityMatrix?): ConcurrentMap { - if (!strmapreturn.containsValue(SMX!!.primaryString)) { - strmapreturn[strmapreturn.size] = SMX.primaryString - val transmittedStr = SMX.secondaryString - val cacheValue1 = SMX.cacheValue1 - val cacheValue2 = SMX.cacheValue2 - if (cacheValue1 != null && !sentimentCachingMap.keys.contains(SMX.primaryString)) { - sentimentCachingMap[SMX.secondaryString] = SMX.cacheValue1 - } - if (cacheValue2 != null && !sentimentCachingMap.keys.contains(transmittedStr)) { - sentimentCachingMap[transmittedStr] = SMX.cacheValue2 - } - } - return strmapreturn - } - - private fun checkForNullValues(index: String?): Boolean { - if (jmweAnnotationCache[index] != null && pipelineAnnotationCache[index] != null - && pipelineSentimentAnnotationCache[index] != null && - coreDocumentAnnotationCache[index] != null) { - return true; - } - return false; - } - - private suspend fun StrComparringNoSentenceRelationMap(strCacheLocal: ConcurrentMap, strCollection: Collection, localJMWEMap: ConcurrentMap, - localPipelineAnnotation: ConcurrentMap, localPipelineSentimentAnnotation: ConcurrentMap, - localCoreDocumentMap: ConcurrentMap): List { - val distance_requirement = 10500.0 - val prefix_size = 150 - val smxReturnList: ArrayList = ArrayList() - - coroutineScope { - for (j in strCollection) { - val job = launch(Dispatchers.Default) { - for (i in strCollection) { - if (j != i) { - val SMXInit = SimilarityMatrix(j, i) - val sentimentCacheStr1 = sentimentCachingMap.getOrDefault(i, null) - val sentimentCacheStr = sentimentCachingMap.getOrDefault(j, null) - var sentimentAnalyzerTest: SentimentAnalyzerTest? = null - val checkedVal: Boolean = checkForNullValues(i) - if (stringCache.size < prefix_size || !checkedVal) { - sentimentAnalyzerTest = SentimentAnalyzerTest(j, i, SMXInit, - localJMWEMap[j], localJMWEMap[i], localPipelineAnnotation[j], - localPipelineAnnotation[i], localPipelineSentimentAnnotation[j], - localPipelineSentimentAnnotation[i], localCoreDocumentMap[j], localCoreDocumentMap[i], - sentimentCacheStr, sentimentCacheStr1) - } else { - sentimentAnalyzerTest = SentimentAnalyzerTest(j, i, SMXInit, - localJMWEMap[j], jmweAnnotationCache[i], localPipelineAnnotation[j], - pipelineAnnotationCache[i], localPipelineSentimentAnnotation[j], - pipelineSentimentAnnotationCache[i], localCoreDocumentMap[j], - coreDocumentAnnotationCache[i], sentimentCacheStr, sentimentCacheStr1) - } - val call = sentimentAnalyzerTest.callSMX(); - if (call != null && call.distance > distance_requirement) { - smxReturnList.add(call) - } - } - } - yield() - } - job.join() - } - } - - return smxReturnList - } - - private suspend fun stringIteratorComparator(strmap: ConcurrentMap, - strCacheLocal: ConcurrentMap, localJMWEMap: ConcurrentMap, - localPipelineAnnotation: ConcurrentMap, localPipelineSentimentAnnotation: ConcurrentMap, - localCoreDocumentMap: ConcurrentMap): ConcurrentMap { - //System.out.println("strmap siuze: " + strmap.size()); - val ComparringNoSentenceRelationMap: List = StrComparringNoSentenceRelationMap(strCacheLocal, strmap.values, - localJMWEMap, localPipelineAnnotation, localPipelineSentimentAnnotation, localCoreDocumentMap) as List - Collections.sort(ComparringNoSentenceRelationMap, Comparator { e1: SimilarityMatrix, e2: SimilarityMatrix -> e1.primaryString.compareTo(e2.primaryString) }) - System.out.println("ComparringNoSentenceRelationMap size: " + ComparringNoSentenceRelationMap.size); - return futuresReturnOverallEvaluation(ComparringNoSentenceRelationMap) - } - - private suspend fun removeNonSensicalStrings(strmap: ConcurrentMap): ConcurrentMap { - val strCacheLocal = stringCache - val localJMWEMap = getMultipleJMWEAnnotation(strmap.values) - val localPipelineAnnotation = getMultiplePipelineAnnotation(strmap.values) - System.out.println("str size post getMultiplePipelineAnnotation: " + strmap.size) - val localPipelineSentimentAnnotation = getMultiplePipelineSentimentAnnotation(strmap.values) - val localCoreDocumentMap = getMultipleCoreDocumentsWaySuggestion(strmap.values, pipeline) - System.out.println("strmap size pre stringIteratorComparator: " + strmap.size) - return stringIteratorComparator(strmap, strCacheLocal, localJMWEMap, localPipelineAnnotation, localPipelineSentimentAnnotation, localCoreDocumentMap) - } - - fun checkIfUpdateStrings() = runBlocking { - if (stopwatch.elapsed(TimeUnit.SECONDS) >= EXPIRE_TIME_IN_SECONDS || !stopwatch.isRunning) { - var str = MessageResponseHandler.getStr() - println("str size: " + str.size) - str = filterContent(str) - System.out.println("str size post filtercontent: " + str.size) - str = removeNonSensicalStrings(str) - System.out.println("removeNonSensicalStrings str size POST: " + str.size + "\n"); - str = annotationCacheUpdate(str) - println(""" - annotationCacheUpdate str size POST: ${str.size} - """.trimIndent()) - val strf = str - if (!stringCache.isEmpty()) { - coroutineScope { - val job = launch(Dispatchers.IO) { - try { - DataMapper.InsertMYSQLStrings(strf) - } catch (ex: CustomError) { - Logger.getLogger(Datahandler::class.java - .name).log(Level.SEVERE, null, ex) - } - MessageResponseHandler.setStr(MapMaker().concurrencyLevel(6).makeMap()) - yield() - } - job.join() - } - } else { - try { - DataMapper.InsertMYSQLStrings(strf) - } catch (ex: CustomError) { - Logger.getLogger(Datahandler::class.java - .name).log(Level.SEVERE, null, ex) - } - MessageResponseHandler.setStr(MapMaker().concurrencyLevel(6).makeMap()) - } + if (stopwatch.elapsed(TimeUnit.MINUTES) >= EXPIRE_TIME_IN_MINUTES || !stopwatch.isRunning) { if (!stopwatch.isRunning) { stopwatch.start() } else { stopwatch.reset() } + stringCache.sortWith(Comparator.comparingInt(String::length).reversed()); + System.out.println("pre InsertMYSQLStrings") + DataMapper.InsertMYSQLStrings(stringCache) } } + fun initiateMYSQL() { + stringCache.addAll(DataMapper.getAllStrings()) + } + private fun trimString(str: String): String { - var str = str - str = str.trim { it <= ' ' } - if (str.startsWith("<@")) { - str = str.substring(str.indexOf("> ") + 2) + var message = str.trim { it <= ' ' } + if (message.startsWith("<@")) { + message = message.substring(message.indexOf("> ") + 2) } - return str + if (!message.isEmpty()) { + message = message.replace("@", "") + if (message.contains("<>")) { + message = message.substring(message.indexOf(">")) + } + if (message.startsWith("[ *")) { + message = message.substring(message.indexOf("]")) + } + } + return message } - private suspend fun getResponseFutures(strF: String): String { - val values_copy: List = ArrayList(stringCache.values) - Collections.sort(values_copy) { o1, o2 -> o2.length - o1.length } - var preRelationUserCounters = -155000.0 - val concurrentRelations: MutableList = arrayListOf() - val SB = StringBuilder() - coroutineScope { - for (str1 in values_copy) { - if (strF != str1) { - val job = launch(Dispatchers.Default) { - var sentimentCacheStr1 = sentimentCachingMap.getOrDefault(str1, null) - var sentimentAnalyzerTest = SentimentAnalyzerTest(strF, str1, SimilarityMatrix(strF, str1), - strAnnoJMWE, jmweAnnotationCache[str1], strAnno, - pipelineAnnotationCache[str1], strAnnoSentiment, - pipelineSentimentAnnotationCache[str1], coreDoc, coreDocumentAnnotationCache[str1], - null, sentimentCacheStr1) + private fun createStrAnnotation(str: String, stanfordCoreNLP: StanfordCoreNLP, sentimentBool: Boolean) { + val strAnno2 = Annotation(str) + strAnno2.compact() + stanfordCoreNLP.annotate(strAnno2) + if (sentimentBool) { + pipelineSentimentAnnotationCache.put(str, strAnno2) + } else { + pipelineAnnotationCache.put(str, strAnno2) + } + } - var getSMX: SimilarityMatrix = sentimentAnalyzerTest.callSMX() - if (getSMX != null) { - val scoreRelationLastUserMsg = getSMX.distance - if (scoreRelationLastUserMsg > preRelationUserCounters) { - preRelationUserCounters = scoreRelationLastUserMsg - concurrentRelations.add(getSMX.secondaryString) - } - } - yield() - } - job.join() + private fun getResponseFutures(strF: String, stanfordCoreNLP: StanfordCoreNLP, stanfordCoreNLPSentiment: StanfordCoreNLP): String { + val strAnno: Annotation = Annotation(strF) + strAnno.compact() + stanfordCoreNLP.annotate(strAnno) + + val strAnnoSentiment: Annotation = Annotation(strF) + strAnnoSentiment.compact() + stanfordCoreNLPSentiment.annotate(strAnnoSentiment) + + val coreDocument = CoreDocument(strF) + stanfordCoreNLP.annotate(coreDocument) + + val values_copy: List = ArrayList(stringCache) + var preRelationUserCounters = -155000.0 + val concurrentRelations: MutableList = arrayListOf() + val SB = StringBuilder() + var jmweAnnotationF = PipelineJMWESingleton.INSTANCE.getJMWEAnnotation(strF) + var tokenizeCountingF: Int? = null + var taggedWordListF: List>? = null + var retrieveTGWListF: java.util.ArrayList? = null + var sentencesF: List? = null + var sentencesSentimentF: List? = null + var coreMaps1: List = jmweAnnotationF.get(CoreAnnotations.SentencesAnnotation::class.java) + var treesF: java.util.ArrayList? = null + + + for (str1 in values_copy) { + if (strF != str1) { + val annotation2 = pipelineSentimentAnnotationCache.getOrDefault(str1, null) + val annotation4 = pipelineAnnotationCache.getOrDefault(str1, null) + val coreDocument1 = coreDocumentAnnotationCache.getOrDefault(str1, null) + var jmweAnnotation = jmweAnnotationCache.getOrDefault(str1, null) + if (annotation2 == null) { + createStrAnnotation(str1, stanfordCoreNLPSentiment, true) + } + if (annotation4 == null) { + createStrAnnotation(str1, stanfordCoreNLP, false) + } + if (coreDocument1 == null) { + getCoreDocumentsSuggested(stanfordCoreNLP, str1) + } + if (jmweAnnotation == null) { + getJMWEAnnotation(str1) + jmweAnnotation = jmweAnnotationCache.get(str1) + } + val tokenizeCounting: Int? = tokenizeCountingHashMap.getOrDefault(str1, null) + val taggedWordList1: List>? = taggedWordListHashMap.getOrDefault(str1, null) + val retrieveTGWList1: java.util.ArrayList? = retrieveTGWListHashMap.getOrDefault(str1, null) + val sentence1: List? = sentences1HashMap.getOrDefault(str1, null) + val sentenceSentiment1: List? = sentencesSentimentHashMap.getOrDefault(str1, null) + val trees1 = trees1HashMap.getOrDefault(str1, null) + var coreMaps2: List = listOf() + if (jmweAnnotation != null) { + coreMaps2 = jmweAnnotation.get(CoreAnnotations.SentencesAnnotation::class.java) + } + var SMX = SentimentAnalyzerTest(strF, str1, SimilarityMatrix(strF, str1), + coreMaps1, coreMaps2, strAnno, + pipelineAnnotationCache[str1], strAnnoSentiment, + pipelineSentimentAnnotationCache[str1], coreDocument, coreDocumentAnnotationCache[str1], + tagger, gsf, classifier, tokenizeCounting, tokenizeCountingF, + taggedWordListF, taggedWordList1, retrieveTGWListF, retrieveTGWList1, + sentencesF, sentence1, sentencesSentimentF, sentenceSentiment1, treesF, trees1) + if (tokenizeCounting == null) { + val tokenizeCounting1 = SMX.getTokenizeCounting(); + tokenizeCountingHashMap.put(str1, tokenizeCounting1) + } + if (taggedWordList1 == null) { + val taggedWordList1Local = SMX.getTaggedWordList1(); + taggedWordListHashMap.put(str1, taggedWordList1Local) + } + if (tokenizeCountingF == null) { + val tokenizeCountingF1 = SMX.getTokenizeCountingF(); + tokenizeCountingF = tokenizeCountingF1; + } + if (taggedWordListF == null) { + val taggedWordListF1 = SMX.getTaggedWordListF(); + taggedWordListF = taggedWordListF1; + } + if (retrieveTGWListF == null) { + val retrieveTGWListF1 = SMX.getRetrieveTGWListF(); + retrieveTGWListF = retrieveTGWListF1; + } + if (retrieveTGWList1 == null) { + val retrieveTGWList11 = SMX.getRetrieveTGWList1(); + retrieveTGWListHashMap.put(str1, retrieveTGWList11); + } + if (sentencesF == null) { + val sentencesF1 = SMX.getSentencesF(); + sentencesF = sentencesF1; + } + if (sentence1 == null) { + val sentences1 = SMX.getSentences1(); + sentences1HashMap.put(str1, sentences1) + } + if (sentencesSentimentF == null) { + val sentencesSentimentF1 = SMX.getSentencesSentimentF(); + sentencesSentimentF = sentencesSentimentF1; + } + if (sentenceSentiment1 == null) { + val sentencesSentiment1 = SMX.getSentencesSentiment1(); + sentencesSentimentHashMap.put(str1, sentencesSentiment1); + } + if (treesF == null) { + val treesF1 = SMX.getTreesF(); + treesF = treesF1; + } + if (trees1 == null) { + val trees11 = SMX.getTrees1(); + trees1HashMap.put(str1, trees11) + } + + var getSMX: SimilarityMatrix = SMX.callSMX() + val scoreRelationLastUserMsg = getSMX.distance + if (scoreRelationLastUserMsg > preRelationUserCounters) { + preRelationUserCounters = scoreRelationLastUserMsg + concurrentRelations.add(getSMX.secondaryString) } } - val randomLenghtPermit = strF.length * (Math.random() * Math.random() * Math.random() * (Math.random() * 10)) - Collections.reverse(concurrentRelations) - if (!concurrentRelations.isEmpty()) { - val firstRelation = concurrentRelations[0] - - val job1 = launch(Dispatchers.Default) { - for (secondaryRelation in concurrentRelations) { - if (SB.toString().length > randomLenghtPermit && !SB.toString().isEmpty()) { - break - } - val append = appendToString(firstRelation, secondaryRelation) - if (append) { - SB.append(secondaryRelation).append(" ") - } - } - yield() + } + val cacheRequirement = 6500; + if (preRelationUserCounters > cacheRequirement && !stringCache.contains(strF) && filterContent(strF)) { + stringCache.add(strF) + } + val randomLenghtPermit = strF.length * (Math.random() * Math.random() * Math.random() * (Math.random() * 10)) + Collections.reverse(concurrentRelations) + if (!concurrentRelations.isEmpty()) { + for (secondaryRelation in concurrentRelations) { + if (SB.toString().length > randomLenghtPermit && !SB.toString().isEmpty()) { + break } - job1.join() + SB.append(secondaryRelation).append(" ") } } if (SB.toString().isEmpty()) { @@ -381,299 +306,79 @@ public class Datahandler { return SB.toString() } - private fun appendToString(firstRelation: String?, secondaryRelation: String?): Boolean { - if (firstRelation == secondaryRelation) { - return true - } - val scoreRelationStrF = getScoreRelationStrF(firstRelation, secondaryRelation) - return if (scoreRelationStrF > 1900) { - true - } else false + private fun getJMWEAnnotation(str1: String) { + val jmweAnnotation = PipelineJMWESingleton.INSTANCE.getJMWEAnnotation(str1) + jmweAnnotationCache.put(str1, jmweAnnotation) } - fun getResponseMsg(str: String): String { - val responseFutures: String + fun getResponseMsg(str: String, personName: String, stanfordCoreNLP: StanfordCoreNLP, + stanfordCoreNLPSentiment: StanfordCoreNLP, ingameResponse: Boolean): String { + var responseFutures: String = "" runBlocking { - val strF = trimString(str) - getSingularAnnotation(strF) - responseFutures = getResponseFutures(strF) + val launch1 = launch(Dispatchers.Default) { + var strF = trimString(str) + responseFutures = getResponseFutures(strF, stanfordCoreNLP, stanfordCoreNLPSentiment) + if (!ingameResponse) { + responseFutures = checkPersonPresentInSentence(personName, responseFutures, strF, stanfordCoreNLP, + stanfordCoreNLPSentiment) + } + yield() + } + launch1.join() } return responseFutures } - suspend fun getSingularAnnotation(str: String?) { - coroutineScope { - val job = launch(Dispatchers.Default) { - strAnno = Annotation(str) - strAnno!!.compact() - pipeline.annotate(strAnno) - yield() - } - job.join() - - val job1 = launch(Dispatchers.Default) { - strAnnoSentiment = Annotation(str) - strAnnoSentiment!!.compact() - pipelineSentiment!!.annotate(strAnnoSentiment) - val notactualList: MutableList = arrayListOf() - notactualList.add(str) - val jmweAnnotation = PipelineJMWESingleton.INSTANCE.getJMWEAnnotation(notactualList) - strAnnoJMWE = jmweAnnotation.values.iterator().next() - strAnnoJMWE.compact() - yield() - } - job1.join() - - val job3 = launch(Dispatchers.Default) { - val coreDocument = CoreDocument(str) - pipeline.annotate(coreDocument) - coreDoc = coreDocument - yield() - } - job3.join() - } - } - - private fun getScoreRelationStrF(str: String?, mostRecentMsg: String?): Double { - val SMX = SimilarityMatrix(str, mostRecentMsg) - val cacheSentiment1 = sentimentCachingMap.getOrDefault(str, null) - val cacheSentiment2 = sentimentCachingMap.getOrDefault(mostRecentMsg, null) - val sentimentAnalyzerTest = SentimentAnalyzerTest(str, mostRecentMsg, SMX, - strAnnoJMWE, jmweAnnotationCache[mostRecentMsg], strAnno, - pipelineAnnotationCache[mostRecentMsg], strAnnoSentiment, - pipelineSentimentAnnotationCache[mostRecentMsg], coreDoc, - coreDocumentAnnotationCache[mostRecentMsg], - cacheSentiment1, cacheSentiment2) - val callSMX = sentimentAnalyzerTest.callSMX() - return callSMX.distance ?: 0.0 - } - - suspend private fun annotationCacheUpdate(strmap: ConcurrentMap): ConcurrentMap { - val jmweAnnotation = PipelineJMWESingleton.INSTANCE.getJMWEAnnotation(strmap.values) - for ((key, value) in jmweAnnotation) { - jmweAnnotationCache[key] = value - } - val Annotationspipeline = MapMaker().concurrencyLevel(5).makeMap() - val AnnotationspipelineSentiment = MapMaker().concurrencyLevel(5).makeMap() - val coreDocumentpipelineMap = getMultipleCoreDocumentsWaySuggestion(strmap.values, pipeline) - - coroutineScope { - val job = launch(Dispatchers.Default) { - for (str in strmap.values) { - val strAnno1 = Annotation(str) - Annotationspipeline[str] = strAnno1 - val strAnno2 = Annotation(str) - AnnotationspipelineSentiment[str] = strAnno2 - stringCache[stringCache.size + 1] = str - } - yield() - } - pipeline.annotate(Annotationspipeline.values, 5) - pipelineSentiment!!.annotate(AnnotationspipelineSentiment.values, 5) - job.join() - } - coroutineScope { - val job = launch(Dispatchers.Default) { - for (pipelineEntry in Annotationspipeline.entries) { - if (pipelineEntry != null) { - pipelineAnnotationCache[pipelineEntry.key] = pipelineEntry.value - } - } - yield() - } - job.join() - } - coroutineScope { - val job = launch(Dispatchers.Default) { - for (coreDocumentEntry in coreDocumentpipelineMap.entries) { - coreDocumentAnnotationCache[coreDocumentEntry.key] = coreDocumentEntry.value - } - yield() - } - job.join() - } - coroutineScope { - val job1 = launch(Dispatchers.Default) { - for (pipelineEntry in AnnotationspipelineSentiment.entries) { - if (pipelineEntry != null) { - pipelineSentimentAnnotationCache[pipelineEntry.key] = pipelineEntry.value - } - } - yield() - } - job1.join() - } - return strmap - } - - - private class AnnotationCollector : Consumer { - val annotationsT: MutableList = arrayListOf() - override fun accept(ann: T) { - //System.out.println("adding ann: " + ann.toString()); - annotationsT.add(ann) - } - - companion object { - var i = 0 - } - } - - companion object { - val EXPIRE_TIME_IN_SECONDS = TimeUnit.SECONDS.convert(10, TimeUnit.MINUTES) - val EXPIRE_TIME_IN_SECONDS1 = TimeUnit.SECONDS.convert(10, TimeUnit.HOURS) - - @JvmField - var instance = Datahandler() - private var strAnno: Annotation? = null - private var strAnnoSentiment: Annotation? = null - private lateinit var strAnnoJMWE: Annotation - private var coreDoc: CoreDocument? = null - private val stringCache = MapMaker().concurrencyLevel(6).makeMap() - private lateinit var pipelineAnnotationCache: ConcurrentMap - private lateinit var pipelineSentimentAnnotationCache: ConcurrentMap - private lateinit var jmweAnnotationCache: ConcurrentMap - private lateinit var coreDocumentAnnotationCache: ConcurrentMap - private val sentimentCachingMap = MapMaker().concurrencyLevel(6).makeMap() - private const val similar = "" - private const val shiftReduceParserPath = "edu/stanford/nlp/models/srparser/englishSR.ser.gz" - private const val sentimentModel = "edu/stanford/nlp/models/sentiment/sentiment.ser.gz" - private const val lexParserEnglishRNN = "edu/stanford/nlp/models/lexparser/englishRNN.ser.gz" - private const val taggerPath = "edu/stanford/nlp/models/pos-tagger/english-left3words/english-left3words-distsim.tagger" - private const val nerModel = "edu/stanford/nlp/models/ner/english.all.3class.caseless.distsim.crf.ser.gz" - private const val nerModel2 = "edu/stanford/nlp/models/ner/english.conll.4class.caseless.distsim.crf.ser.gz" - private const val nerModel3 = "edu/stanford/nlp/models/ner/english.muc.7class.caseless.distsim.crf.ser.gz" - private const val customStopWordList = "start,starts,period,periods,a,an,and,are,as,at,be,but,by,for,if,in,into,is,it,no,not,of,on,or,such,that,the,their,then,there,these,they,this,to,was,will,with" - - @JvmStatic - var tagger: MaxentTagger? = null - private set - private val options = arrayOf("-maxLength", "100") - private val props = Properties() - private val propsSentiment = Properties() - - @JvmStatic - var gsf: GrammaticalStructureFactory? = null - private set - private lateinit var lp: LexicalizedParser - private lateinit var tlp: TreebankLanguagePack - private lateinit var classifier: AbstractSequenceClassifier - - public fun getPipeLine(): StanfordCoreNLP { - return pipeline - } - - // set up Stanford CoreNLP pipeline - @JvmStatic - val pipeline = pipeLineSetUp - private var pipelineSentiment: StanfordCoreNLP? = null - - private val pipeLineSetUp: StanfordCoreNLP - private get() { - props.setProperty("annotators", "tokenize,ssplit,pos,lemma,ner,parse") - props.setProperty("parse.model", shiftReduceParserPath) - props.setProperty("parse.maxlen", "90") - props.setProperty("parse.binaryTrees", "true") - props.setProperty("threads", "5") - props.setProperty("pos.maxlen", "90") - props.setProperty("tokenize.maxlen", "90") - props.setProperty("ssplit.maxlen", "90") - props.setProperty("lemma.maxlen", "90") - props.setProperty("ner.model", "$nerModel,$nerModel2,$nerModel3") - props.setProperty("ner.combinationMode", "HIGH_RECALL") - props.setProperty("regexner.ignorecase", "true") - props.setProperty("ner.fine.regexner.ignorecase", "true") - props.setProperty("tokenize.options", "untokenizable=firstKeep") - return StanfordCoreNLP(props) - } - - @JvmStatic - fun getClassifier(): AbstractSequenceClassifier { - return classifier - } - - private fun getMultipleJMWEAnnotation(str: Collection): ConcurrentMap { - return PipelineJMWESingleton.INSTANCE.getJMWEAnnotation(str) - } - - private fun getMultiplePipelineAnnotation(str: Collection): ConcurrentMap { - val pipelineAnnotationMap = MapMaker().concurrencyLevel(5).makeMap() - for (str1 in str) { - val strAnno1 = Annotation(str1) - pipelineAnnotationMap[str1] = strAnno1 - } - pipeline.annotate(pipelineAnnotationMap.values, 5) - return pipelineAnnotationMap - } - - private fun getMultiplePipelineSentimentAnnotation(str: Collection): ConcurrentMap { - val pipelineAnnotationMap = MapMaker().concurrencyLevel(5).makeMap() - for (str1 in str) { - val strAnno1 = Annotation(str1) - pipelineAnnotationMap[str1] = strAnno1 - } - pipelineSentiment?.annotate(pipelineAnnotationMap.values, 5) - return pipelineAnnotationMap - } - - fun filterContent(str: ConcurrentMap): ConcurrentMap { - val strlistreturn = MapMaker().concurrencyLevel(5).makeMap() - for (str1: String? in str.values) { - if (!str1?.isEmpty()!! && str1.length > 3) { - var str1Local: String = str1.trim(); - if (str1Local.length > 2 && !str1Local.startsWith("!")) { - strlistreturn[strlistreturn.size] = str1Local - } - } - } - return strlistreturn - } - - suspend fun getMultipleCoreDocumentsWaySuggestion(str: Collection, localNLP: StanfordCoreNLP): ConcurrentMap { - val annCollector: AnnotationCollector = AnnotationCollector() - val annotationreturnMap = MapMaker().concurrencyLevel(6).makeMap() - coroutineScope { - val job = launch(Dispatchers.Default) { - for (exampleString in str) { - localNLP.annotate(Annotation(exampleString), annCollector) - AnnotationCollector.i++ - } - yield() - } - job.join() - } - try { - Thread.sleep(1500) - } catch (ex: InterruptedException) { - Logger.getLogger(Datahandler::class.java.name).log(Level.SEVERE, null, ex) - } - coroutineScope { - val job1 = launch(Dispatchers.Default) { - for (ann in annCollector.annotationsT) { - if (ann != null) { - ann.compact() - val CD = CoreDocument(ann) - annotationreturnMap[CD.text()] = CD + private fun checkPersonPresentInSentence(personName: String, responseMsg: String, userLastMessage: String, + stanfordCoreNLP: StanfordCoreNLP, + stanfordCoreNLPSentiment: StanfordCoreNLP): String { + try { + val pipelineCoreDcoument = CoreDocument(responseMsg) + val pipelineCoreDcoumentLastMsg = CoreDocument(userLastMessage) + stanfordCoreNLP.annotate(pipelineCoreDcoument) + stanfordCoreNLPSentiment.annotate(pipelineCoreDcoumentLastMsg) + val regex = "(.*?\\d){10,}" + for (em in pipelineCoreDcoument.entityMentions()) { + val entityType = em.entityType() + if (entityType == "PERSON") { + var str = responseMsg + val emText = em.text() + val pattern = Pattern.compile(regex) + val matcher = pattern.matcher(personName) + val isMatched = matcher.matches() + if (emText != personName && !isMatched) { + for (emLastMsg in pipelineCoreDcoumentLastMsg.entityMentions()) { + if (emText != emLastMsg.text() && !Character.isDigit(emLastMsg.text().trim { it <= ' ' }[0])) { + //System.out.println("emLastMsg.text(): " + emLastMsg.text()); + str = (responseMsg.substring(0, responseMsg.indexOf(emText)) + " " + + emLastMsg + " " + responseMsg.substring(responseMsg.indexOf(emText))) + } } + str += " $personName" + return str } - yield() } - job1.join() } - try { - Thread.sleep(1500) - } catch (ex: InterruptedException) { - Logger.getLogger(Datahandler::class.java.name).log(Level.SEVERE, null, ex) - } - return annotationreturnMap + } catch (e: Exception) { + println("""SCUFFED JAYZ: ${e.localizedMessage}""".trimIndent()) } + return responseMsg } - init { - stopwatch = Stopwatch.createUnstarted() - jmweAnnotationCache = MapMaker().concurrencyLevel(5).makeMap() - pipelineAnnotationCache = MapMaker().concurrencyLevel(5).makeMap() - pipelineSentimentAnnotationCache = MapMaker().concurrencyLevel(5).makeMap() - coreDocumentAnnotationCache = MapMaker().concurrencyLevel(5).makeMap() + fun filterContent(str: String): Boolean { + if (!str.isEmpty() && str.length > 3) { + var str1Local: String = str.trim(); + if (str1Local.length > 2 && !str1Local.startsWith("!")) { + return true + } + } + return false + } + + fun getCoreDocumentsSuggested(pipeline: StanfordCoreNLP, str: String) { + val annotation = Annotation(str) + pipeline.annotate(annotation) + val coreDocument = CoreDocument(annotation) + coreDocumentAnnotationCache.put(str, coreDocument) } } \ No newline at end of file diff --git a/ArtificialAutism/src/main/java/FunctionLayer/DoStuff.java b/ArtificialAutism/src/main/java/FunctionLayer/DoStuff.java index 403c3288..e1df79f9 100644 --- a/ArtificialAutism/src/main/java/FunctionLayer/DoStuff.java +++ b/ArtificialAutism/src/main/java/FunctionLayer/DoStuff.java @@ -16,6 +16,7 @@ import java.util.List; import java.util.logging.Level; import java.util.logging.Logger; +import edu.stanford.nlp.pipeline.StanfordCoreNLP; import reactor.core.publisher.Flux; import reactor.core.publisher.Mono; @@ -25,30 +26,26 @@ import reactor.core.publisher.Mono; */ public class DoStuff { - public static boolean occupied = false; - public static boolean isOccupied() { - return occupied; - } - - public static void doStuff(MessageCreateEvent event, String usernameBot) { - String username = null; + public static void doStuff(MessageCreateEvent event, String usernameBot, Datahandler datahandler, + StanfordCoreNLP stanfordCoreNLP, StanfordCoreNLP stanfordCoreNLPSentiment) { + String username = ""; try { username = event.getMessage().getAuthor().get().getUsername(); } catch (java.util.NoSuchElementException e) { username = null; } if (username != null && !username.equals(usernameBot)) { - occupied = true; TextChannel block = event.getMessage().getChannel().cast(TextChannel.class).block(); String name = block.getCategory().block().getName(); name = name.toLowerCase(); String channelName = block.getName().toLowerCase(); boolean channelpermissionsDenied = false; + if (channelName.contains("suggestion-box")) { + channelpermissionsDenied = true; + } switch (name) { - case "public area": { - break; - } + case "public area": case "information area": { break; } @@ -57,37 +54,36 @@ public class DoStuff { break; } } - List blockLast = event.getMessage().getUserMentions().buffer().blockLast(); - String content = event.getMessage().getContent(); if (!channelpermissionsDenied) { + List blockLast = event.getMessage().getUserMentions().buffer().blockLast(); + String content = event.getMessage().getContent(); if (blockLast != null) { for (User user : blockLast) { content = content.replace(user.getId().asString(), ""); } } - MessageResponseHandler.getMessage(content); - } - boolean mentionedBot = false; - if (blockLast != null) { - for (User user : blockLast) { - if (user.getUsername().equals(usernameBot)) { - mentionedBot = true; - break; + boolean mentionedBot = false; + if (blockLast != null) { + for (User user : blockLast) { + if (user.getUsername().equals(usernameBot)) { + mentionedBot = true; + break; + } + } + } + if (mentionedBot || channelName.contains("general-autism")) { + String ResponseStr; + ResponseStr = datahandler.getResponseMsg(content, username, stanfordCoreNLP, stanfordCoreNLPSentiment, + false); + if (!ResponseStr.isEmpty()) { + System.out.print("\nResponseStr3: " + ResponseStr + "\n"); + event.getMessage().getChannel().block().createMessage(ResponseStr).block(); } } } - if (mentionedBot || channelName.contains("general-autism")) { - String ResponseStr; - ResponseStr = MessageResponseHandler.selectReponseMessage(content, username); - if (!ResponseStr.isEmpty()) { - System.out.print("\nResponseStr3: " + ResponseStr + "\n"); - event.getMessage().getChannel().block().createMessage(ResponseStr).block(); - } - } new Thread(() -> { - Datahandler.instance.checkIfUpdateStrings(); + datahandler.updateStringCache(); }).start(); - occupied = false; } } } diff --git a/ArtificialAutism/src/main/java/FunctionLayer/MessageResponseHandler.java b/ArtificialAutism/src/main/java/FunctionLayer/MessageResponseHandler.java deleted file mode 100644 index 01acd727..00000000 --- a/ArtificialAutism/src/main/java/FunctionLayer/MessageResponseHandler.java +++ /dev/null @@ -1,99 +0,0 @@ -/* - * To change this license header, choose License Headers in Project Properties. - * To change this template file, choose Tools | Templates - * and open the template in the editor. - */ -package FunctionLayer; - -import com.google.common.collect.MapMaker; -import edu.stanford.nlp.pipeline.CoreDocument; -import edu.stanford.nlp.pipeline.CoreEntityMention; - -import java.util.ArrayList; -import java.util.Collection; -import java.util.Collections; -import java.util.List; -import java.util.concurrent.ConcurrentMap; -import java.util.regex.Matcher; -import java.util.regex.Pattern; - -/** - * @author install1 - */ -public class MessageResponseHandler { - - private static ConcurrentMap str = new MapMaker().concurrencyLevel(6).makeMap(); - - public static ConcurrentMap getStr() { - ArrayList arrayList = new ArrayList(str.values()); - Collections.sort(arrayList, (o1, o2) -> o2.length() - o1.length()); - int iterator = 0; - for (String str1 : arrayList) { - str.put(iterator, str1); - iterator++; - } - return str; - } - - public static void setStr(ConcurrentMap str) { - MessageResponseHandler.str = str; - } - - public static void getMessage(String message) { - if (message != null && !message.isEmpty()) { - message = message.replace("@", ""); - if (message.contains("<>")) { - message = message.substring(message.indexOf(">")); - } - if (message.startsWith("[ *")) { - message = message.substring(message.indexOf("]")); - } - str.put(str.size() + 1, message); - } - } - - public static String selectReponseMessage(String toString, String personName) { - String getResponseMsg = Datahandler.instance.getResponseMsg(toString); - getResponseMsg = checkPersonPresentInSentence(personName, getResponseMsg, toString); - return getResponseMsg; - } - - private static String checkPersonPresentInSentence(String personName, String responseMsg, String userLastMessage) { - //check if userlastmsg contains person as refference - //check if first person is author or their person of mention - try { - String strreturn = responseMsg; - CoreDocument pipelineCoreDcoument = new CoreDocument(responseMsg); - CoreDocument pipelineCoreDcoumentLastMsg = new CoreDocument(userLastMessage); - Datahandler.getPipeline().annotate(pipelineCoreDcoument); - Datahandler.getPipeline().annotate(pipelineCoreDcoumentLastMsg); - //Datahandler.pipeline.annotate(pipelineCoreDcoument); - //Datahandler.pipeline.annotate(pipelineCoreDcoumentLastMsg); - String regex = "(.*?\\d){10,}"; - for (CoreEntityMention em : pipelineCoreDcoument.entityMentions()) { - String entityType = em.entityType(); - if (entityType.equals("PERSON")) { - String str = strreturn; - String emText = em.text(); - Pattern pattern = Pattern.compile(regex); - Matcher matcher = pattern.matcher(personName); - boolean isMatched = matcher.matches(); - if (!emText.equals(personName) && !isMatched) { - for (CoreEntityMention emLastMsg : pipelineCoreDcoumentLastMsg.entityMentions()) { - if (!emText.equals(emLastMsg.text()) && !Character.isDigit(emLastMsg.text().trim().charAt(0))) { - //System.out.println("emLastMsg.text(): " + emLastMsg.text()); - str = strreturn.substring(0, strreturn.indexOf(emText)) + " " - + emLastMsg + " " + strreturn.substring(strreturn.indexOf(emText)); - } - } - str += " " + personName; - return str; - } - } - } - } catch (Exception e) { - System.out.println("SCUFFED JAYZ: " + e.getLocalizedMessage() + "\n"); - } - return responseMsg; - } -} diff --git a/ArtificialAutism/src/main/java/FunctionLayer/PipelineJMWESingleton.java b/ArtificialAutism/src/main/java/FunctionLayer/PipelineJMWESingleton.java index 84337a65..778cdb61 100644 --- a/ArtificialAutism/src/main/java/FunctionLayer/PipelineJMWESingleton.java +++ b/ArtificialAutism/src/main/java/FunctionLayer/PipelineJMWESingleton.java @@ -5,7 +5,6 @@ */ package FunctionLayer; -import com.google.common.collect.MapMaker; import edu.mit.jmwe.data.IMWE; import edu.mit.jmwe.data.IToken; import edu.mit.jmwe.data.Token; @@ -28,10 +27,8 @@ import edu.stanford.nlp.util.CoreMap; import java.io.File; import java.io.IOException; import java.util.ArrayList; -import java.util.Collection; import java.util.List; import java.util.Properties; -import java.util.concurrent.ConcurrentMap; /** * @author install1 @@ -40,20 +37,14 @@ import java.util.concurrent.ConcurrentMap; public class PipelineJMWESingleton { //if not needed to be volatile dont make it, increases time - public volatile static PipelineJMWESingleton INSTANCE; + //public volatile static PipelineJMWESingleton INSTANCE; + public static PipelineJMWESingleton INSTANCE; private static StanfordCoreNLP localNLP = initializeJMWE(); private static String underscoreSpaceReplacement; + private static IMWEIndex index; + private static IMWEDetector detector; private PipelineJMWESingleton() { - } - - public static void getINSTANCE() { - INSTANCE = new PipelineJMWESingleton(); - } - - public final ConcurrentMap getJMWEAnnotation(Collection strvalues) { - boolean verbose = false; - IMWEIndex index; String jmweIndexData = "/home/gameservers/autism_bot/lib/mweindex_wordnet3.0_semcor1.6.data"; // ./lib/mweindex_wordnet3.0_semcor1.6.data String jmweIndexDataLocalTest = "E:/java8/Projects/mweindex_wordnet3.0_semcor1.6.data"; File indexFile = new File((String) jmweIndexData); @@ -64,22 +55,30 @@ public class PipelineJMWESingleton { } catch (IOException e) { throw new RuntimeException("unable to open IMWEIndex index: " + e + "\n"); } - IMWEDetector detector = getDetector(index, detectorName); - ConcurrentMap returnAnnotations = new MapMaker().concurrencyLevel(5).makeMap(); - strvalues.forEach(str -> { - Annotation annoStr = new Annotation(str); - returnAnnotations.put(str, annoStr); - }); - localNLP.annotate(returnAnnotations.values(), 4); - - returnAnnotations.values().parallelStream().forEach(annoStr -> { - for (CoreMap sentence : annoStr.get(CoreAnnotations.SentencesAnnotation.class)) { - List> mwes = getjMWEInSentence(sentence, index, detector, verbose); - sentence.set(JMWEAnnotation.class, mwes); - } - }); + detector = getDetector(index, detectorName); index.close(); - return returnAnnotations; + } + + public static void getINSTANCE() { + INSTANCE = new PipelineJMWESingleton(); + } + + public final Annotation getJMWEAnnotation(String str) { + try { + index.open(); + } catch (IOException e) { + throw new RuntimeException("unable to open IMWEIndex index: " + e + "\n"); + } + Annotation annoStr = new Annotation(str); + localNLP.annotate(annoStr); + Class sentencesAnnotationClass = CoreAnnotations.SentencesAnnotation.class; + for (CoreMap sentence : annoStr.get(sentencesAnnotationClass)) { + List> mwes = getjMWEInSentence(sentence, index, detector, false); + //annoStr.set(JMWEAnnotation.class, mwes); + sentence.set(JMWEAnnotation.class, mwes); + } + index.close(); + return annoStr; } public final static StanfordCoreNLP initializeJMWE() { @@ -94,7 +93,7 @@ public class PipelineJMWESingleton { propsJMWE.setProperty("lemma.maxlen", "90"); underscoreSpaceReplacement = "-"; localNLP = new StanfordCoreNLP(propsJMWE); - System.out.println("finished singleton constructor \n"); + System.out.println("finished JMWE constructor \n"); return localNLP; } @@ -147,5 +146,4 @@ public class PipelineJMWESingleton { } return sentence; } - } diff --git a/ArtificialAutism/src/main/java/FunctionLayer/SimilarityMatrix.java b/ArtificialAutism/src/main/java/FunctionLayer/SimilarityMatrix.java index 23ada343..59154703 100644 --- a/ArtificialAutism/src/main/java/FunctionLayer/SimilarityMatrix.java +++ b/ArtificialAutism/src/main/java/FunctionLayer/SimilarityMatrix.java @@ -5,10 +5,7 @@ */ package FunctionLayer; -import FunctionLayer.StanfordParser.SentimentValueCache; - /** - * * @author install1 */ public class SimilarityMatrix { @@ -16,8 +13,6 @@ public class SimilarityMatrix { private String PrimaryString; private String SecondaryString; private double distance; - private SentimentValueCache cacheValue1; - private SentimentValueCache cacheValue2; public final double getDistance() { return distance; @@ -38,36 +33,8 @@ public class SimilarityMatrix { this.distance = result; } - public final String getPrimaryString() { - return PrimaryString; - } - - public final void setPrimaryString(String PrimaryString) { - this.PrimaryString = PrimaryString; - } - public final String getSecondaryString() { return SecondaryString; } - public final void setSecondaryString(String SecondaryString) { - this.SecondaryString = SecondaryString; - } - - public final SentimentValueCache getCacheValue1() { - return cacheValue1; - } - - public final void setCacheValue1(SentimentValueCache cacheValue1) { - this.cacheValue1 = cacheValue1; - } - - public final SentimentValueCache getCacheValue2() { - return cacheValue2; - } - - public final void setCacheValue2(SentimentValueCache cacheValue2) { - this.cacheValue2 = cacheValue2; - } - } diff --git a/ArtificialAutism/src/main/java/FunctionLayer/StanfordParser/SentimentAnalyzerTest.java b/ArtificialAutism/src/main/java/FunctionLayer/StanfordParser/SentimentAnalyzerTest.java index 797f87f2..3fe79975 100644 --- a/ArtificialAutism/src/main/java/FunctionLayer/StanfordParser/SentimentAnalyzerTest.java +++ b/ArtificialAutism/src/main/java/FunctionLayer/StanfordParser/SentimentAnalyzerTest.java @@ -1,10 +1,8 @@ package FunctionLayer.StanfordParser; import FunctionLayer.LevenshteinDistance; -import FunctionLayer.Datahandler; import FunctionLayer.SimilarityMatrix; import FunctionLayer.StopwordAnnotator; -import com.google.common.collect.MapMaker; import edu.mit.jmwe.data.IMWE; import edu.mit.jmwe.data.IMWEDesc; import edu.mit.jmwe.data.IToken; @@ -19,7 +17,6 @@ import edu.stanford.nlp.neural.rnn.RNNCoreAnnotations; import edu.stanford.nlp.pipeline.Annotation; import edu.stanford.nlp.pipeline.CoreDocument; import edu.stanford.nlp.pipeline.CoreEntityMention; -import edu.stanford.nlp.pipeline.StanfordCoreNLP; import edu.stanford.nlp.process.CoreLabelTokenFactory; import edu.stanford.nlp.process.DocumentPreprocessor; import edu.stanford.nlp.process.PTBTokenizer; @@ -39,17 +36,7 @@ import edu.stanford.nlp.util.CoreMap; import edu.stanford.nlp.util.Pair; import java.io.StringReader; -import java.util.AbstractMap; -import java.util.ArrayList; -import java.util.Collection; -import java.util.List; -import java.util.Map; -import java.util.Objects; -import java.util.OptionalDouble; -import java.util.Set; -import java.util.concurrent.Callable; -import java.util.concurrent.ConcurrentMap; -import java.util.concurrent.atomic.AtomicInteger; +import java.util.*; import org.apache.lucene.analysis.core.StopAnalyzer; import org.ejml.simple.SimpleMatrix; @@ -71,36 +58,115 @@ public class SentimentAnalyzerTest { private final MaxentTagger tagger; private final GrammaticalStructureFactory gsf; private final AbstractSequenceClassifier classifier; - private final Annotation jmweStrAnnotation1; - private final Annotation jmweStrAnnotation2; + private final List coreMaps1; + private final List coreMaps2; private final Annotation pipelineAnnotation1; private final Annotation pipelineAnnotation2; private final Annotation pipelineAnnotation1Sentiment; private final Annotation pipelineAnnotation2Sentiment; private final CoreDocument pipelineCoreDcoument1; private final CoreDocument pipelineCoreDcoument2; - private SentimentValueCache cacheSentiment1; - private SentimentValueCache cacheSentiment2; - public SentimentAnalyzerTest(String str, String str1, SimilarityMatrix smxParam, Annotation str1Annotation, Annotation str2Annotation, + public Integer getTokenizeCounting() { + return tokenizeCounting; + } + + public List> getTaggedWordListF() { + return taggedWordListF; + } + + public List> getTaggedWordList1() { + return taggedWordList1; + } + + public ArrayList getRetrieveTGWList1() { + return retrieveTGWList1; + } + + public List getSentencesF() { + return sentencesF; + } + + public Integer getTokenizeCountingF() { + return tokenizeCountingF; + } + + public ArrayList getRetrieveTGWListF() { + return retrieveTGWListF; + } + + public List getSentences1() { + return sentences1; + } + + public List getSentencesSentimentF() { + return sentencesSentimentF; + } + + public List getSentencesSentiment1() { + return sentencesSentiment1; + } + + public ArrayList getTreesF() { + return treesF; + } + + public ArrayList getTrees1() { + return trees1; + } + + //caches + private Integer tokenizeCounting; + private Integer tokenizeCountingF; + private List> taggedWordListF; + private List> taggedWordList1; + private ArrayList retrieveTGWList1; + private ArrayList retrieveTGWListF; + private List sentencesF; + private List sentences1; + private List sentencesSentimentF; + private List sentencesSentiment1; + private ArrayList treesF; + private ArrayList trees1; + + + public SentimentAnalyzerTest(String str, String str1, SimilarityMatrix smxParam, List coreMaps1, List coreMaps2, Annotation strPipeline1, Annotation strPipeline2, Annotation strPipeSentiment1, Annotation strPipeSentiment2, - CoreDocument pipelineCoreDcoument1, CoreDocument pipelineCoreDcoument2, SentimentValueCache cacheValue1, SentimentValueCache cacheValue2) { + CoreDocument pipelineCoreDcoument1, CoreDocument pipelineCoreDcoument2, + MaxentTagger tagger, GrammaticalStructureFactory gsf, + AbstractSequenceClassifier classifier, Integer tokenizeCounting, + Integer tokenizeCountingF, List> taggedWordListF, + List> taggedWordList1, ArrayList + retrieveTGWListF, ArrayList retrieveTGWList1, + List sentencesF, List sentences1, + List sentencesSentimentF, List sentencesSentiment1, + ArrayList treesF, ArrayList trees1) { this.str = str; this.str1 = str1; this.smxParam = smxParam; - this.tagger = Datahandler.getTagger(); - this.gsf = Datahandler.getGsf(); - this.classifier = Datahandler.getClassifier(); - this.jmweStrAnnotation1 = str1Annotation; - this.jmweStrAnnotation2 = str2Annotation; + this.tagger = tagger; + this.gsf = gsf; + this.classifier = classifier; + this.coreMaps1 = coreMaps1; + this.coreMaps2 = coreMaps2; this.pipelineAnnotation1 = strPipeline1; this.pipelineAnnotation2 = strPipeline2; this.pipelineAnnotation1Sentiment = strPipeSentiment1; this.pipelineAnnotation2Sentiment = strPipeSentiment2; this.pipelineCoreDcoument1 = pipelineCoreDcoument1; this.pipelineCoreDcoument2 = pipelineCoreDcoument2; - this.cacheSentiment1 = cacheValue1; - this.cacheSentiment2 = cacheValue2; + this.tokenizeCounting = tokenizeCounting; + this.tokenizeCountingF = tokenizeCountingF; + this.taggedWordListF = taggedWordListF; + this.taggedWordList1 = taggedWordList1; + this.retrieveTGWListF = retrieveTGWListF; + this.retrieveTGWList1 = retrieveTGWList1; + this.sentencesF = sentencesF; + this.sentences1 = sentences1; + this.sentencesSentimentF = sentencesSentimentF; + this.sentencesSentiment1 = sentencesSentiment1; + this.treesF = treesF; + this.trees1 = trees1; } private List> getTaggedWordList(String message) { @@ -122,13 +188,13 @@ public class SentimentAnalyzerTest { return counter; } - private ConcurrentMap retrieveTGWListIndex(List> taggedwordlist) { - ConcurrentMap tgwlistIndex = new MapMaker().concurrencyLevel(5).makeMap(); + private ArrayList retrieveTGWListIndex(List> taggedwordlist) { + ArrayList tgwlistIndex = new ArrayList(); for (List tGWList : taggedwordlist) { for (TaggedWord taggedWord : tGWList) { - for (String str : tgwlistIndex.values()) { + for (String str : tgwlistIndex) { if (!taggedWord.tag().equals(str) && !taggedWord.tag().equals(":")) { - tgwlistIndex.put(tgwlistIndex.size() + 1, taggedWord.tag()); + tgwlistIndex.add(taggedWord.tag()); tGWList.remove(taggedWord); } } @@ -137,30 +203,30 @@ public class SentimentAnalyzerTest { return tgwlistIndex; } - private Double iterateTrees(ConcurrentMap sentenceConstituencyParseList2, ConcurrentMap sentenceConstituencyParseList1, + private Double iterateTrees(ArrayList sentenceConstituencyParseList2, ArrayList sentenceConstituencyParseList1, Double score) { double preConstituentsScore = score; - ConcurrentMap constituentsMap = new MapMaker().concurrencyLevel(5).makeMap(); + ArrayList constituentsMap = new ArrayList(); int constituencySize = sentenceConstituencyParseList1.size() + sentenceConstituencyParseList2.size(); - for (final Tree sentenceConstituencyParse2 : sentenceConstituencyParseList2.values()) { + for (final Tree sentenceConstituencyParse2 : sentenceConstituencyParseList2) { int constiRelationsize = 0; - for (final Tree sentenceConstituencyParse1 : sentenceConstituencyParseList1.values()) { + for (final Tree sentenceConstituencyParse1 : sentenceConstituencyParseList1) { Set constinuent1 = Tdiff.markDiff(sentenceConstituencyParse1, sentenceConstituencyParse2); Set constinuent2 = Tdiff.markDiff(sentenceConstituencyParse2, sentenceConstituencyParse1); - ConcurrentMap constiLabels = new MapMaker().concurrencyLevel(5).makeMap(); + ArrayList constiLabels = new ArrayList(); for (final Constituent consti : constinuent1) { for (final Constituent consti1 : constinuent2) { - if (consti.value().equals(consti1.value()) && !constiLabels.values().contains(consti.value())) { - constiLabels.put(constiLabels.size(), consti.value()); + if (consti.value().equals(consti1.value()) && !constiLabels.contains(consti.value())) { + constiLabels.add(consti.value()); constiRelationsize++; } } } int constituents1 = constinuent1.size() - constiRelationsize; int constituents2 = constinuent2.size() - constiRelationsize; - constituentsMap.put(constituentsMap.size(), constituents1); - constituentsMap.put(constituentsMap.size(), constituents2); - constituentsMap.put(constituentsMap.size(), constiRelationsize); + constituentsMap.add(constituents1); + constituentsMap.add(constituents2); + constituentsMap.add(constiRelationsize); if (constituentsMap.size() < 4) { if ((constituents1 * 5 < constituents2 || constituents2 * 5 < constituents1) && constituents1 > 0 && constituents2 > 0) { score -= (constituents1 + constituents2) * 200; @@ -219,10 +285,10 @@ public class SentimentAnalyzerTest { } private Double typeDependenciesGrammaticalRelation(Collection allTypedDependencies1, Collection allTypedDependencies2, - Double score, ConcurrentMap grammaticalMap1, ConcurrentMap grammaticalMap2, - ConcurrentMap sentenceConstituencyParseList1, ConcurrentMap sentenceConstituencyParseList2) { - ConcurrentMap alltypeDepsSizeMap = new MapMaker().concurrencyLevel(5).makeMap(); - ConcurrentMap summationMap = new MapMaker().concurrencyLevel(5).makeMap(); + Double score, ArrayList grammaticalMap1, ArrayList grammaticalMap2, + ArrayList sentenceConstituencyParseList1, ArrayList sentenceConstituencyParseList2) { + ArrayList alltypeDepsSize1 = new ArrayList(); + ArrayList summationList = new ArrayList(); int relationApplicable1 = 0; int relationApplicable2 = 0; int grammaticalRelation1 = 0; @@ -232,9 +298,9 @@ public class SentimentAnalyzerTest { for (TypedDependency TDY1 : allTypedDependencies1) { IndexedWord dep = TDY1.dep(); IndexedWord gov = TDY1.gov(); - for (GrammaticalStructure gs : grammaticalMap1.values()) { + for (GrammaticalStructure gs : grammaticalMap1) { GrammaticalRelation grammaticalRelation = gs.getGrammaticalRelation(gov, dep); - for (Tree sentenceConstituencyParse2 : sentenceConstituencyParseList2.values()) { + for (Tree sentenceConstituencyParse2 : sentenceConstituencyParseList2) { if (grammaticalRelation.isApplicable(sentenceConstituencyParse2) && !treeCollectionGramatical.contains(sentenceConstituencyParse2)) { score += 700; grammaticalRelation1++; @@ -254,9 +320,9 @@ public class SentimentAnalyzerTest { for (TypedDependency TDY : allTypedDependencies2) { IndexedWord dep = TDY.dep(); IndexedWord gov = TDY.gov(); - for (GrammaticalStructure gs : grammaticalMap2.values()) { + for (GrammaticalStructure gs : grammaticalMap2) { GrammaticalRelation grammaticalRelation = gs.getGrammaticalRelation(gov, dep); - for (Tree sentenceConstituencyParse1 : sentenceConstituencyParseList1.values()) { + for (Tree sentenceConstituencyParse1 : sentenceConstituencyParseList1) { if (grammaticalRelation.isApplicable(sentenceConstituencyParse1) && !treeCollectionGramatical.contains(sentenceConstituencyParse1)) { score += 700; grammaticalRelation2++; @@ -289,8 +355,8 @@ public class SentimentAnalyzerTest { } else { score -= allTypeDep1 > allTypeDep2 ? (allTypeDep1 - allTypeDep2) * 600 : (allTypeDep2 - allTypeDep1) * 600; } - alltypeDepsSizeMap.put(alltypeDepsSizeMap.size() + 1, allTypeDep1); - alltypeDepsSizeMap.put(alltypeDepsSizeMap.size() + 1, allTypeDep2); + alltypeDepsSize1.add(allTypeDep1); + alltypeDepsSize1.add(allTypeDep2); } } if (allTypeDep1 >= 5 && allTypeDep2 >= 5) { @@ -298,9 +364,9 @@ public class SentimentAnalyzerTest { int smallerTypeDep = allTypeDep1 < allTypeDep2 ? allTypeDep1 : allTypeDep2; int summation = (largerTypeDep * largerTypeDep) - (smallerTypeDep * smallerTypeDep); if (summation / largerTypeDep < 15.0 && summation / largerTypeDep > 10.0 && smallerTypeDep * 2 > largerTypeDep - && !summationMap.values().contains(summation)) { + && !summationList.contains(summation)) { score += summation * 80; - summationMap.put(summationMap.size() + 1, summation); + summationList.add(summation); } else if (largerTypeDep == smallerTypeDep) { score += 2500; } @@ -328,33 +394,33 @@ public class SentimentAnalyzerTest { : (grammaticalRelation2 - grammaticalRelation1) * 500; } } - ConcurrentMap filerTreeContent = new MapMaker().concurrencyLevel(5).makeMap(); - AtomicInteger runCount1 = new AtomicInteger(0); - for (Tree sentenceConstituencyParse1 : sentenceConstituencyParseList1.values()) { - for (Tree sentenceConstituencyParse2 : sentenceConstituencyParseList2.values()) { + ArrayList filerTreeContent = new ArrayList(); + int runCount1 = 0; + for (Tree sentenceConstituencyParse1 : sentenceConstituencyParseList1) { + for (Tree sentenceConstituencyParse2 : sentenceConstituencyParseList2) { for (CoreLabel LBW : sentenceConstituencyParse1.taggedLabeledYield()) { for (CoreLabel LBW1 : sentenceConstituencyParse2.taggedLabeledYield()) { if (LBW.lemma().equals(LBW1.lemma())) { boolean found = false; - for (String str : filerTreeContent.values()) { + for (String str : filerTreeContent) { if (str.equals(LBW.lemma())) { found = true; break; } } if (!found) { - filerTreeContent.put(filerTreeContent.size() + 1, LBW.lemma()); - runCount1.getAndIncrement(); + filerTreeContent.add(LBW.lemma()); + runCount1++; } } } } } } - score += runCount1.get() * 250; + score += runCount1 * 250; int typeSizeSmallest = 100; int typeSizeLargest = 0; - for (Integer i : alltypeDepsSizeMap.values()) { + for (Integer i : alltypeDepsSize1) { if (i > typeSizeLargest) { typeSizeLargest = i; } @@ -367,7 +433,7 @@ public class SentimentAnalyzerTest { } typeSizeLargest = 0; typeSizeSmallest = 100; - for (int i : summationMap.values()) { + for (int i : summationList) { if (i > typeSizeLargest) { typeSizeLargest = i; } @@ -381,54 +447,55 @@ public class SentimentAnalyzerTest { return score; } - private Double simpleRNNMatrixCalculations(Double score, ConcurrentMap simpleSMXlist1, ConcurrentMap simpleSMXlist2) { + private Double simpleRNNMatrixCalculations(Double score, ArrayList simpleSMXlist1, + ArrayList simpleSMXlist2) { List iteratedDoubleList = new ArrayList(); List iterateddotPredictions = new ArrayList(); double dotpredictionTransfer = 0.0; int iterationOverHeat = 0; double scoreFallback = score; - for (SimpleMatrix simpleSMX2 : simpleSMXlist2.values()) { - ConcurrentMap AccumulateDotMap = new MapMaker().concurrencyLevel(5).makeMap(); - ConcurrentMap subtractorMap = new MapMaker().concurrencyLevel(5).makeMap(); - ConcurrentMap dotPredictions = new MapMaker().concurrencyLevel(5).makeMap(); - ConcurrentMap DotOverTransfer = new MapMaker().concurrencyLevel(5).makeMap(); + for (SimpleMatrix simpleSMX2 : simpleSMXlist2) { + ArrayList AccumulateDotList = new ArrayList<>(); + ArrayList subtractorList = new ArrayList(); + ArrayList dotPredictions = new ArrayList(); + ArrayList DotOverTransfer = new ArrayList(); Double totalSubtraction = 0.0; Double largest = 10.0; Double shortest = 100.0; - for (SimpleMatrix simpleSMX1 : simpleSMXlist1.values()) { + for (SimpleMatrix simpleSMX1 : simpleSMXlist1) { double dotPrediction2 = simpleSMX2.dot(simpleSMX1) * 100; double dotPrediction1 = simpleSMX1.dot(simpleSMX2) * 100; - AccumulateDotMap.put(AccumulateDotMap.size() + 1, dotPrediction1); - AccumulateDotMap.put(AccumulateDotMap.size() + 1, dotPrediction2); + AccumulateDotList.add(dotPrediction1); + AccumulateDotList.add(dotPrediction2); double subtracter1 = dotPrediction1 > 50 ? dotPrediction1 - 100 : dotPrediction1 > 0 ? 100 - dotPrediction1 : 0; double subtracter2 = dotPrediction2 > 50 ? dotPrediction2 - 100 : dotPrediction2 > 0 ? 100 - dotPrediction2 : 0; - subtractorMap.put(subtractorMap.size() + 1, subtracter1); - subtractorMap.put(subtractorMap.size() + 1, subtracter2); + subtractorList.add(subtracter1); + subtractorList.add(subtracter2); dotpredictionTransfer = dotPrediction1; - if (!dotPredictions.values().contains(dotPrediction1)) { - for (Double transferDots : DotOverTransfer.values()) { + if (!dotPredictions.contains(dotPrediction1)) { + for (Double transferDots : DotOverTransfer) { if (transferDots == dotPrediction1) { totalSubtraction += transferDots; } else { score -= subtracter1 * 25; } } - DotOverTransfer.put(DotOverTransfer.size(), dotPrediction1); + DotOverTransfer.add(dotPrediction1); } else { subtracter1 -= 100; subtracter1 *= 25; score += subtracter1 * dotPrediction1; } - dotPredictions.put(dotPredictions.size() + 1, dotPrediction1); - if (!dotPredictions.values().contains(dotPrediction2)) { - for (Double transferDots : DotOverTransfer.values()) { + dotPredictions.add(dotPrediction1); + if (!dotPredictions.contains(dotPrediction2)) { + for (Double transferDots : DotOverTransfer) { if (transferDots == dotPrediction2) { totalSubtraction += transferDots; } else { score -= subtracter1 * 25; } } - DotOverTransfer.put(DotOverTransfer.size(), dotPrediction2); + DotOverTransfer.add(dotPrediction2); if (dotPrediction2 > largest) { largest = dotPrediction2; } @@ -438,7 +505,7 @@ public class SentimentAnalyzerTest { Double dotPredictionIntervalDifference = largest - shortest; subtracter2 *= 25; if (dotPredictionIntervalDifference < 5.0) { - if (dotPredictions.values().size() > 0) { + if (dotPredictions.size() > 0) { if (subtracter2 > 0) { score -= subtracter2; } else { @@ -465,7 +532,7 @@ public class SentimentAnalyzerTest { score -= 4500; } } else if (!iterateddotPredictions.contains(dotPrediction2)) { - score -= subtracter2 * dotPrediction2; // += + score += subtracter2 * dotPrediction2; // += iterateddotPredictions.add(dotPrediction2); } else { score -= 550; @@ -485,19 +552,19 @@ public class SentimentAnalyzerTest { } } } - dotPredictions.put(dotPredictions.size() + 1, dotPrediction2); + dotPredictions.add(dotPrediction2); iterationOverHeat++; } Double subTracPre = 0.0; - for (Double subtractors : subtractorMap.values()) { + for (Double subtractors : subtractorList) { if (Objects.equals(subTracPre, subtractors)) { if (subTracPre > 43.5 && subTracPre < 50.0) { - score += (subTracPre * 15) / subtractorMap.values().size(); + score += (subTracPre * 15) / subtractorList.size(); } else if (subTracPre > 60.0 && subTracPre < 66.5) { - score += (subTracPre * 15) / subtractorMap.values().size(); + score += (subTracPre * 15) / subtractorList.size(); } } else if (subTracPre > 75.0 && subTracPre < 90.0) { - score += (subTracPre * 50) / subtractorMap.values().size(); + score += (subTracPre * 50) / subtractorList.size(); } else if (subTracPre >= 55.0) { score -= 2800; } else if (subTracPre < -25.0 && subTracPre > -45.0) { @@ -512,7 +579,7 @@ public class SentimentAnalyzerTest { } Double preAccumulatorDot = 0.0; Double postAccumulatorDot = 0.0; - for (Double accumulators : AccumulateDotMap.values()) { + for (Double accumulators : AccumulateDotList) { if (Objects.equals(preAccumulatorDot, accumulators)) { if (Objects.equals(postAccumulatorDot, accumulators)) { score -= 1400; @@ -522,7 +589,7 @@ public class SentimentAnalyzerTest { preAccumulatorDot = accumulators; } subTracPre = 0.0; - for (Double subtractors : subtractorMap.values()) { + for (Double subtractors : subtractorList) { if (Objects.equals(subTracPre, subtractors) && subTracPre != 0.0) { if (!iteratedDoubleList.contains(subTracPre)) { score += 500; @@ -541,16 +608,16 @@ public class SentimentAnalyzerTest { return score; } - private Double simpleRNNMaxtrixVectors(Double score, ConcurrentMap simpleSMXlistVector1, ConcurrentMap simpleSMXlistVector2) { - ConcurrentMap elementSumCounter = new MapMaker().concurrencyLevel(5).makeMap(); - ConcurrentMap dotMap = new MapMaker().concurrencyLevel(5).makeMap(); - ConcurrentMap elementSumMap = new MapMaker().concurrencyLevel(5).makeMap(); - ConcurrentMap dotSumMap = new MapMaker().concurrencyLevel(5).makeMap(); + private Double simpleRNNMaxtrixVectors(Double score, ArrayList simpleSMXlistVector1, ArrayList simpleSMXlistVector2) { + ArrayList elementSumCounter = new ArrayList<>(); + ArrayList dotMap = new ArrayList<>(); + ArrayList elementSumMap = new ArrayList<>(); + ArrayList dotSumMap = new ArrayList<>(); Double preDot = 0.0; Double postDot = 0.0; - int iterateSize = simpleSMXlistVector1.values().size() + simpleSMXlistVector2.values().size(); - for (SimpleMatrix simpleSMX2 : simpleSMXlistVector2.values()) { - for (SimpleMatrix simpleSMX1 : simpleSMXlistVector1.values()) { + int iterateSize = simpleSMXlistVector1.size() + simpleSMXlistVector2.size(); + for (SimpleMatrix simpleSMX2 : simpleSMXlistVector2) { + for (SimpleMatrix simpleSMX1 : simpleSMXlistVector1) { double dot2 = simpleSMX2.dot(simpleSMX1); double elementSum2 = simpleSMX2.kron(simpleSMX1).elementSum(); double dot1 = simpleSMX1.dot(simpleSMX2); @@ -569,13 +636,13 @@ public class SentimentAnalyzerTest { } preDot = dot1; elementSum1 = Math.round(elementSum1 * 100.0) / 100.0; - elementSumCounter.put(elementSumCounter.size() + 1, elementSum1); - dotMap.put(dotMap.size() + 1, dot1); + elementSumCounter.add(elementSum1); + dotMap.add(dot1); preDot = dot2; elementSum2 = Math.round(elementSum2 * 100.0) / 100.0; - elementSumCounter.put(elementSumCounter.size() + 1, elementSum2); - dotMap.put(dotMap.size() + 1, dot2); - if (!dotSumMap.values().contains(dot1)) { + elementSumCounter.add(elementSum2); + dotMap.add(dot2); + if (!dotSumMap.contains(dot1)) { if (dot1 < 0.1 && dot1 > 0.050) { score += 256; } @@ -584,11 +651,11 @@ public class SentimentAnalyzerTest { } else if (dot1 > 0.40 && dot1 < 0.445) { score += 3600; } - dotSumMap.put(dotSumMap.size() + 1, dot1); + dotSumMap.add(dot1); } else { score -= 50; } - if (!elementSumMap.values().contains(elementSum1)) { + if (!elementSumMap.contains(elementSum1)) { if (elementSum1 < 0.01 && elementSum1 > 0.00) { score += 1300; } else if (elementSum1 > 0.1 && elementSum1 < 1.0) { @@ -596,11 +663,11 @@ public class SentimentAnalyzerTest { } else { score -= elementSum1 * 1024; } - elementSumMap.put(elementSumMap.size() + 1, elementSum1); + elementSumMap.add(elementSum1); } else { score -= 50; } - if (!dotSumMap.values().contains(dot2)) { + if (!dotSumMap.contains(dot2)) { if (dot2 < 0.000) { score += dot2 * 500; } else if (dot2 < 0.1) { @@ -609,13 +676,13 @@ public class SentimentAnalyzerTest { if (dot2 > 0.50) { score -= 1200; } - dotSumMap.put(dotSumMap.size() + 1, dot2); + dotSumMap.add(dot2); } else if (dot2 > 0.050 && dot2 < 0.10) { score -= 350; } else { score = score > 0 ? score - dot2 * 1200 : score + dot2 * 1200; } - if (!elementSumMap.values().contains(elementSum2)) { + if (!elementSumMap.contains(elementSum2)) { if (elementSum2 < 0.01 && elementSum2 > 0.00) { score += 3300; } else if (elementSum2 > 0.1 && elementSum2 < 0.2) { @@ -623,7 +690,7 @@ public class SentimentAnalyzerTest { } else { score -= elementSum2 * 1024; } - elementSumMap.put(elementSumMap.size() + 1, elementSum2); + elementSumMap.add(elementSum2); } else if (elementSum2 > 0.050 && elementSum2 < 0.10) { score += 750; } else { @@ -638,10 +705,9 @@ public class SentimentAnalyzerTest { return score; } - private Double elementsAndDotsRelation(Double score, ConcurrentMap dotMap, ConcurrentMap elementSumCounter) { - OptionalDouble minvalueDots = dotMap.values().stream().mapToDouble(Double::doubleValue).min(); - OptionalDouble maxvalueDots = dotMap.values().stream().mapToDouble(Double::doubleValue).max(); - double total = minvalueDots.getAsDouble() + maxvalueDots.getAsDouble(); + private Double elementsAndDotsRelation(Double score, ArrayList dotMap, ArrayList elementSumCounter) { + OptionalDouble minvalueDots = dotMap.stream().mapToDouble(Double::doubleValue).min(); + OptionalDouble maxvalueDots = dotMap.stream().mapToDouble(Double::doubleValue).max(); boolean permitted = false; if (minvalueDots.getAsDouble() != maxvalueDots.getAsDouble()) { permitted = true; @@ -659,8 +725,8 @@ public class SentimentAnalyzerTest { score -= 3500; } } - OptionalDouble minvalueElements = elementSumCounter.values().stream().mapToDouble(Double::doubleValue).min(); - OptionalDouble maxvalueElements = elementSumCounter.values().stream().mapToDouble(Double::doubleValue).max(); + OptionalDouble minvalueElements = elementSumCounter.stream().mapToDouble(Double::doubleValue).min(); + OptionalDouble maxvalueElements = elementSumCounter.stream().mapToDouble(Double::doubleValue).max(); Double elementsVariance = maxvalueElements.getAsDouble() - minvalueElements.getAsDouble(); if (elementsVariance != 0.0) { if (elementsVariance <= 0.01 && maxvalueElements.getAsDouble() <= 0.02) { @@ -721,28 +787,18 @@ public class SentimentAnalyzerTest { return score; } - private final Map.Entry> classifyRawEvaluation(Double score, SentimentValueCache cacheSentimentLocal1, - SentimentValueCache cacheSentimentLocal2) { - if (cacheSentiment1 == null || cacheSentiment2 == null) { - DocumentReaderAndWriter readerAndWriter = classifier.makePlainTextReaderAndWriter(); - if (cacheSentiment1 == null) { - cacheSentimentLocal1.setClassifyRaw(classifier.classifyRaw(str, readerAndWriter)); - } - if (cacheSentiment2 == null) { - cacheSentimentLocal2.setClassifyRaw(classifier.classifyRaw(str1, readerAndWriter)); - } - } - final List classifyRaw1 = cacheSentiment1 == null ? cacheSentimentLocal1.getClassifyRaw() : cacheSentiment1.getClassifyRaw(); - final List classifyRaw2 = cacheSentiment2 == null ? cacheSentimentLocal2.getClassifyRaw() : cacheSentiment2.getClassifyRaw(); - score -= (classifyRaw1.size() > classifyRaw2.size() ? classifyRaw1.size() - classifyRaw2.size() : classifyRaw2.size() - classifyRaw1.size()) * 200; - Map.Entry> entry - = new AbstractMap.SimpleEntry(score, new AbstractMap.SimpleEntry(cacheSentimentLocal1, cacheSentimentLocal2)); - return entry; + private int classifyRawEvaluation() { + DocumentReaderAndWriter readerAndWriter = classifier.makePlainTextReaderAndWriter(); + List list1 = classifier.classifyRaw(str, readerAndWriter); + List list2 = classifier.classifyRaw(str1, readerAndWriter); + final List classifyRaw1 = list1; + final List classifyRaw2 = list2; + return (classifyRaw1.size() > classifyRaw2.size() ? classifyRaw1.size() - classifyRaw2.size() : classifyRaw2.size() - classifyRaw1.size()) * 200; } - private Double entryCountsRelation(Double score, SentimentValueCache cacheSentimentLocal1, SentimentValueCache cacheSentimentLocal2) { - int entry1 = cacheSentiment1 == null ? cacheSentimentLocal1.getEntryCounts().values().size() : cacheSentiment1.getEntryCounts().values().size(); - int entry2 = cacheSentiment2 == null ? cacheSentimentLocal2.getEntryCounts().values().size() : cacheSentiment2.getEntryCounts().values().size(); + private Double entryCountsRelation(Double score, ArrayList cacheSentimentLocal1, ArrayList cacheSentimentLocal2) { + int entry1 = cacheSentimentLocal1.size(); + int entry2 = cacheSentimentLocal2.size(); if (entry1 > 0 && entry2 > 0) { if ((entry1 >= entry2 * 5) || (entry2 >= entry1 * 5)) { score -= entry1 > entry2 ? (entry1 - entry2) * 450 : (entry2 - entry1) * 450; @@ -751,7 +807,7 @@ public class SentimentAnalyzerTest { } else if ((entry1 >= entry2 * 2 || entry2 >= entry1 * 2) && entry1 * 2 >= entry2 && entry2 * 2 >= entry1) { score -= entry1 > entry2 ? (entry1 - entry2) * 450 : (entry2 - entry1) * 450; } else if (entry1 * 3 >= entry2 && entry2 * 3 >= entry1) { - score += entry1 > entry2 ? (entry1 - entry2) * 550 : (entry2 - entry1) * 550; + score -= entry1 > entry2 ? (entry1 - entry2) * 550 : (entry2 - entry1) * 550; } else if (entry1 > 10 && entry2 > 10 && entry1 * 2 > entry2 && entry2 * 2 > entry1) { score += entry1 > entry2 ? entry2 * 600 : entry1 * 600; } @@ -759,104 +815,109 @@ public class SentimentAnalyzerTest { return score; } - private SentimentValueCache GrammaticStructureSetup(SentimentValueCache cacheSentimentLocal, Annotation pipelineAnnotation) { - for (CoreMap sentence : pipelineAnnotation.get(CoreAnnotations.SentencesAnnotation.class)) { - Tree sentenceConstituencyParse = sentence.get(TreeCoreAnnotations.TreeAnnotation.class); - cacheSentimentLocal.addSentenceConstituencyParse(sentenceConstituencyParse); - GrammaticalStructure gs = gsf.newGrammaticalStructure(sentenceConstituencyParse); - cacheSentimentLocal.addTypedDependencies(gs.allTypedDependencies()); - cacheSentimentLocal.addGS(gs); + private ArrayList grammaticalStructureAllTypedDependencies( + ArrayList grammaticalStructures) { + ArrayList typedDependenciesArr = new ArrayList<>(); + for (GrammaticalStructure gs : grammaticalStructures) { + Collection typedDependencies = gs.allTypedDependencies(); + typedDependenciesArr.addAll(typedDependencies); } - return cacheSentimentLocal; + return typedDependenciesArr; } - private SentimentValueCache initializeCacheSetup(String str, SentimentValueCache cacheSentimentLocal) { - cacheSentimentLocal = new SentimentValueCache(str); - cacheSentimentLocal.setTaggedwords(getTaggedWordList(str)); - cacheSentimentLocal.setCounter(tokenizeCounting(cacheSentimentLocal.getTaggedwordlist())); - return cacheSentimentLocal; + private ArrayList grammaticalStructureSetup(ArrayList trees) { + ArrayList grammaticalStructures = new ArrayList(); + for (Tree tree : trees) { + GrammaticalStructure gs = gsf.newGrammaticalStructure(tree); + grammaticalStructures.add(gs); + } + return grammaticalStructures; } - private SentimentValueCache sentimentCoreAnnotationSetup(Annotation pipelineAnnotationSentiment, SentimentValueCache cacheSentimentLocal) { - for (CoreMap sentence : pipelineAnnotationSentiment.get(CoreAnnotations.SentencesAnnotation.class)) { - Tree tree = sentence.get(SentimentCoreAnnotations.SentimentAnnotatedTree.class); + private ArrayList retrieveTrees(List sentences) { + ArrayList treeList = new ArrayList(); + for (CoreMap sentence : sentences) { + Tree sentenceConstituencyParse = sentence.get(TreeCoreAnnotations.TreeAnnotation.class); + treeList.add(sentenceConstituencyParse); + } + return treeList; + } + + private ArrayList sentimentRNNCorePredicted(List sentences, + Class sentimentAnnotatedTreeClass) { + ArrayList rnnCoreAnnotationsPrediction = new ArrayList<>(); + for (CoreMap sentence : sentences) { + Tree tree = sentence.get(sentimentAnnotatedTreeClass); + if (tree != null) { + SimpleMatrix predictions = RNNCoreAnnotations.getPredictions(tree); + rnnCoreAnnotationsPrediction.add(predictions); + } + } + return rnnCoreAnnotationsPrediction; + } + + private ArrayList sentimentRNNCoreNodevectors(List sentences, + Class sentimentAnnotatedTreeClass) { + ArrayList rnnCoreAnnotationsNodevectors = new ArrayList<>(); + for (CoreMap sentence : sentences) { + Tree tree = sentence.get(sentimentAnnotatedTreeClass); + if (tree != null) { + SimpleMatrix nodeVector = RNNCoreAnnotations.getNodeVector(tree); + rnnCoreAnnotationsNodevectors.add(nodeVector); + } + } + return rnnCoreAnnotationsNodevectors; + } + + private ArrayList sentimentRNNCoreAnnotations(List sentences, + Class sentimentAnnotatedTreeClass) { + ArrayList rnnCoreAnnotationsPredicted = new ArrayList<>(); + for (CoreMap sentence : sentences) { + Tree tree = sentence.get(sentimentAnnotatedTreeClass); if (tree != null) { int predictedClass = RNNCoreAnnotations.getPredictedClass(tree); - SimpleMatrix predictions = RNNCoreAnnotations.getPredictions(tree); - SimpleMatrix nodeVector = RNNCoreAnnotations.getNodeVector(tree); - cacheSentimentLocal.addRNNPredictClass(predictedClass); - cacheSentimentLocal.addSimpleMatrix(predictions); - cacheSentimentLocal.addSimpleMatrixVector(nodeVector); + rnnCoreAnnotationsPredicted.add(predictedClass); } } - return cacheSentimentLocal; + return rnnCoreAnnotationsPredicted; } - private SentimentValueCache setupMainSentimentandLongestVal(Annotation pipelineAnnotationSentiment, SentimentValueCache cacheSentimentLocal) { - for (CoreMap sentence : pipelineAnnotationSentiment.get(CoreAnnotations.SentencesAnnotation.class)) { - Tree tree = sentence.get(SentimentCoreAnnotations.SentimentAnnotatedTree.class); + private int setupMainSentiment(List sentences4, + Class sentimentAnnotatedTreeClass) { + int longest = 0; + int longestSentiment = 0; + for (CoreMap sentence : sentences4) { + Tree tree = sentence.get(sentimentAnnotatedTreeClass); int sentiment = RNNCoreAnnotations.getPredictedClass(tree); String partText = sentence.toString(); - //SimpleMatrix predictions = RNNCoreAnnotations.getPredictions(tree); - if (partText.length() > cacheSentimentLocal.getLongest()) { - cacheSentimentLocal.setMainSentiment(sentiment); - cacheSentimentLocal.setLongest(partText.length()); + if (partText.length() > longest) { + longestSentiment = sentiment; + longest = partText.length(); } } - return cacheSentimentLocal; + return longestSentiment; } - private SentimentValueCache jmweAnnotationSetup(Annotation jmweStrAnnotation, SentimentValueCache cacheSentimentLocal) { - List sentences = jmweStrAnnotation.get(CoreAnnotations.SentencesAnnotation.class); - Collection> tokeninflectionMap = new ArrayList(); - int tokenadder = 0; + private int setupMainLongest(List sentences) { + int longest = 0; for (CoreMap sentence : sentences) { - for (IMWE token : sentence.get(JMWEAnnotation.class)) { - if (token.isInflected()) { - cacheSentimentLocal.setInflectedCounterPositive(cacheSentimentLocal.getInflectedCounterPositive() + 1); - } else if (!tokeninflectionMap.contains(token)) { - cacheSentimentLocal.setInflectedCounterNegative(cacheSentimentLocal.getInflectedCounterNegative() + 1); - tokeninflectionMap.add(token); - } - cacheSentimentLocal.addstrTokenForm(token.getForm()); - cacheSentimentLocal.addstrTokenGetEntry(token.getEntry().toString().substring(token.getEntry().toString().length() - 1)); - Collection values = token.getPartMap().values(); - IMWEDesc entry = token.getEntry(); - cacheSentimentLocal.setMarkedContinuousCounter(cacheSentimentLocal.getMarkedContinuousCounter() + entry.getMarkedContinuous()); - cacheSentimentLocal.setUnmarkedPatternCounter(cacheSentimentLocal.getUnmarkedPatternCounter() + entry.getUnmarkedPattern()); - for (IMWEDesc.IPart iPart : values) { - cacheSentimentLocal.addstrTokenGetiPart(iPart.getForm()); - } - for (String strPostPrefix : entry.getPOS().getPrefixes()) { - cacheSentimentLocal.addstrTokenEntryPOS(strPostPrefix); - } - for (int counts : entry.getCounts()) { - cacheSentimentLocal.addEntryCounts(counts); - } - for (IToken tokens : token.getTokens()) { - cacheSentimentLocal.addITokenMapTag(tokens.getTag()); - for (String strtoken : tokens.getStems()) { - cacheSentimentLocal.addstrTokenStems(strtoken); - cacheSentimentLocal.setMarkedContiniousCounterEntries(cacheSentimentLocal.getMarkedContiniousCounterEntries() + 1); - } - } - tokenadder += 1; + String partText = sentence.toString(); + if (partText.length() > longest) { + longest = partText.length(); } - cacheSentimentLocal.setAnotatorcounter(cacheSentimentLocal.getAnotatorcounter() + 1); } - cacheSentimentLocal.setTokensCounter(tokenadder); - return cacheSentimentLocal; + return longest; } - private Double entryCountsScoring(Double score, SentimentValueCache cacheSentimentLocal1, SentimentValueCache cacheSentimentLocal2) { - ConcurrentMap countsMap = new MapMaker().concurrencyLevel(5).makeMap(); - int totalsize = cacheSentimentLocal1.getEntryCounts().values().size() + cacheSentimentLocal2.getEntryCounts().values().size(); - for (int counts : cacheSentimentLocal1.getEntryCounts().values()) { - for (int counts1 : cacheSentimentLocal2.getEntryCounts().values()) { + private Double entryCountsScoring(Double score, ArrayList cacheSentimentLocal1, ArrayList cacheSentimentLocal2) { + ArrayList countsMap = new ArrayList(); + int totalsize = cacheSentimentLocal1.size() + cacheSentimentLocal2.size(); + for (int counts : cacheSentimentLocal1) { + for (int counts1 : cacheSentimentLocal2) { if (counts > 0 && counts1 > 0) { - if (counts == counts1 && !countsMap.values().contains(counts)) { + if (counts == counts1 && !countsMap.contains(counts)) { score += (counts * 250) / totalsize; - countsMap.put(countsMap.size() + 1, counts); + countsMap.add(counts); } else if (counts * 3 < counts1 || counts1 * 3 < counts) { score -= 600; } @@ -866,17 +927,17 @@ public class SentimentAnalyzerTest { return score; } - private Double tokenEntryPosScoring(Double score, SentimentValueCache cacheSentimentLocal1, SentimentValueCache cacheSentimentLocal2) { - if (cacheSentimentLocal1.getstrTokenEntryPOS().values().size() > 1 && cacheSentimentLocal2.getstrTokenEntryPOS().values().size() > 1) { - for (String strTokenPos1 : cacheSentimentLocal1.getstrTokenEntryPOS().values()) { - for (String strTokenPos2 : cacheSentimentLocal2.getstrTokenEntryPOS().values()) { + private Double tokenEntryPosScoring(Double score, ArrayList cacheSentimentLocal1, ArrayList cacheSentimentLocal2) { + if (cacheSentimentLocal1.size() > 1 && cacheSentimentLocal2.size() > 1) { + for (String strTokenPos1 : cacheSentimentLocal1) { + for (String strTokenPos2 : cacheSentimentLocal2) { if (strTokenPos1.equals(strTokenPos2)) { score += 500; } } } - int posEntrySize1 = cacheSentimentLocal1.getstrTokenEntryPOS().values().size(); - int posEntrySize2 = cacheSentimentLocal2.getstrTokenEntryPOS().values().size(); + int posEntrySize1 = cacheSentimentLocal1.size(); + int posEntrySize2 = cacheSentimentLocal2.size(); if (posEntrySize1 * 3 > posEntrySize2 && posEntrySize2 * 3 > posEntrySize1) { score += posEntrySize1 > posEntrySize2 ? (posEntrySize1 - posEntrySize2) * 700 : (posEntrySize2 - posEntrySize1) * 700; } @@ -884,9 +945,7 @@ public class SentimentAnalyzerTest { return score; } - private Double unmarkedPatternCounterScoring(Double score, SentimentValueCache cacheSentimentLocal1, SentimentValueCache cacheSentimentLocal2) { - int UnmarkedPatternCounter1 = cacheSentimentLocal1.getUnmarkedPatternCounter(); - int UnmarkedPatternCounter2 = cacheSentimentLocal2.getUnmarkedPatternCounter(); + private Double unmarkedPatternCounterScoring(Double score, int UnmarkedPatternCounter1, int UnmarkedPatternCounter2) { if (UnmarkedPatternCounter1 > 0 && UnmarkedPatternCounter2 > 0) { if (UnmarkedPatternCounter1 < 100 && UnmarkedPatternCounter2 < 100) { if (UnmarkedPatternCounter1 * 2 > UnmarkedPatternCounter2 && UnmarkedPatternCounter2 * 2 > UnmarkedPatternCounter1) { @@ -901,11 +960,9 @@ public class SentimentAnalyzerTest { return score; } - private Double markedContiniousCounterScoring(Double score, SentimentValueCache cacheSentimentLocal1, SentimentValueCache cacheSentimentLocal2) { - int MarkedContinuousCounter1 = cacheSentimentLocal1.getMarkedContinuousCounter(); - int MarkedContinuousCounter2 = cacheSentimentLocal2.getMarkedContinuousCounter(); - int MarkedContiniousCounter1Entries = cacheSentimentLocal1.getMarkedContiniousCounterEntries(); - int MarkedContiniousCounter2Entries = cacheSentimentLocal2.getMarkedContiniousCounterEntries(); + private Double markedContiniousCounterScoring(Double score, int MarkedContinuousCounter1, int MarkedContinuousCounter2) { + int MarkedContiniousCounter1Entries = MarkedContinuousCounter1; + int MarkedContiniousCounter2Entries = MarkedContinuousCounter2; if (MarkedContinuousCounter1 > 0 && MarkedContinuousCounter2 > 0) { if (MarkedContinuousCounter1 > MarkedContinuousCounter2 * 50 || MarkedContinuousCounter2 > MarkedContinuousCounter1 * 50) { score -= MarkedContinuousCounter1 > MarkedContinuousCounter2 ? MarkedContinuousCounter1 * 120 : MarkedContinuousCounter2 * 120; @@ -933,19 +990,19 @@ public class SentimentAnalyzerTest { return score; } - private Double strTokensMapScoring(Double score, SentimentValueCache cacheSentimentLocal1, SentimentValueCache cacheSentimentLocal2) { - ConcurrentMap strtokensMap = new MapMaker().concurrencyLevel(5).makeMap(); - for (String strTokeniPart1 : cacheSentimentLocal1.getstrTokenGetiPart().values()) { - for (String strTokeniPart2 : cacheSentimentLocal2.getstrTokenGetiPart().values()) { - if (strTokeniPart1.equals(strTokeniPart2) && !strtokensMap.values().contains(strTokeniPart2)) { - strtokensMap.put(strtokensMap.size() + 1, strTokeniPart2); + private Double strTokensMapScoring(Double score, ArrayList cacheSentimentLocal1, ArrayList cacheSentimentLocal2) { + ArrayList strtokensMap = new ArrayList(); + for (String strTokeniPart1 : cacheSentimentLocal1) { + for (String strTokeniPart2 : cacheSentimentLocal2) { + if (strTokeniPart1.equals(strTokeniPart2) && !strtokensMap.contains(strTokeniPart2)) { + strtokensMap.add(strTokeniPart2); score += 800; } } } - int tokenIPartSize1 = cacheSentimentLocal1.getstrTokenGetiPart().values().size(); - int tokenIPartSize2 = cacheSentimentLocal2.getstrTokenGetiPart().values().size(); - int strTokenMapSize = strtokensMap.values().size(); + int tokenIPartSize1 = cacheSentimentLocal1.size(); + int tokenIPartSize2 = cacheSentimentLocal2.size(); + int strTokenMapSize = strtokensMap.size(); if (tokenIPartSize1 * 2 > tokenIPartSize2 && tokenIPartSize2 * 2 > tokenIPartSize1) { score += tokenIPartSize1 > tokenIPartSize2 ? (tokenIPartSize1 - tokenIPartSize2) * 700 : (tokenIPartSize2 - tokenIPartSize1) * 700; score += strTokenMapSize * 600; @@ -955,18 +1012,18 @@ public class SentimentAnalyzerTest { return score; } - private Double strTokenEntryScoring(Double score, SentimentValueCache cacheSentimentLocal1, SentimentValueCache cacheSentimentLocal2) { - int tokenEntry1 = cacheSentimentLocal1.getstrTokenGetEntry().values().size(); - int tokenEntry2 = cacheSentimentLocal2.getstrTokenGetEntry().values().size(); + private Double strTokenEntryScoring(Double score, ArrayList cacheSentimentLocal1, ArrayList cacheSentimentLocal2) { + int tokenEntry1 = cacheSentimentLocal1.size(); + int tokenEntry2 = cacheSentimentLocal2.size(); boolean boundaryLeaks = false; int remnantCounter = 0; if (tokenEntry1 * 2 != tokenEntry2 && tokenEntry2 * 2 != tokenEntry1) { boundaryLeaks = true; } - ConcurrentMap entryTokenMap = new MapMaker().concurrencyLevel(5).makeMap(); - for (String strTokenEntry1 : cacheSentimentLocal1.getstrTokenGetEntry().values()) { - for (String strTokenEntry2 : cacheSentimentLocal2.getstrTokenGetEntry().values()) { - if (!entryTokenMap.values().contains(strTokenEntry2)) { + ArrayList entryTokenMap = new ArrayList(); + for (String strTokenEntry1 : cacheSentimentLocal1) { + for (String strTokenEntry2 : cacheSentimentLocal2) { + if (!entryTokenMap.contains(strTokenEntry2)) { if (strTokenEntry1.equals(strTokenEntry2)) { score += boundaryLeaks ? 2500 : 2500 / 2; } else if (!boundaryLeaks) { @@ -975,26 +1032,26 @@ public class SentimentAnalyzerTest { remnantCounter++; } } - entryTokenMap.put(entryTokenMap.size() + 1, strTokenEntry2); + entryTokenMap.add(strTokenEntry2); } } score += remnantCounter * 250; return score; } - private Double strTokenMapTagsScoring(Double score, SentimentValueCache cacheSentimentLocal1, SentimentValueCache cacheSentimentLocal2) { - ConcurrentMap iTokenMapTagsMap = new MapMaker().concurrencyLevel(5).makeMap(); - for (String strmapTag : cacheSentimentLocal1.getITokenMapTag().values()) { - for (String strmapTag1 : cacheSentimentLocal2.getITokenMapTag().values()) { - if (strmapTag.equals(strmapTag1) && !iTokenMapTagsMap.values().contains(strmapTag1)) { + private Double strTokenMapTagsScoring(Double score, ArrayList cacheSentimentLocal1, ArrayList cacheSentimentLocal2) { + ArrayList iTokenMapTagsMap = new ArrayList(); + for (String strmapTag : cacheSentimentLocal1) { + for (String strmapTag1 : cacheSentimentLocal2) { + if (strmapTag.equals(strmapTag1) && !iTokenMapTagsMap.contains(strmapTag1)) { score -= 1450; - iTokenMapTagsMap.put(iTokenMapTagsMap.size() + 1, strmapTag); + iTokenMapTagsMap.add(strmapTag); } } } - int mapTagsize1 = cacheSentimentLocal1.getITokenMapTag().values().size(); - int mapTagsize2 = cacheSentimentLocal2.getITokenMapTag().values().size(); - int tokenTagMapSize = iTokenMapTagsMap.values().size(); + int mapTagsize1 = cacheSentimentLocal1.size(); + int mapTagsize2 = cacheSentimentLocal2.size(); + int tokenTagMapSize = iTokenMapTagsMap.size(); if (mapTagsize1 != 0 && mapTagsize2 != 0) { if (mapTagsize1 * 2 > mapTagsize2 && mapTagsize2 * 2 > mapTagsize1) { score += mapTagsize1 > mapTagsize2 ? (mapTagsize1 - mapTagsize2) * 700 : (mapTagsize2 - mapTagsize1) * 700; @@ -1006,13 +1063,13 @@ public class SentimentAnalyzerTest { return score; } - private Double tokenformSizeScoring(Double score, SentimentValueCache cacheSentimentLocal1, SentimentValueCache cacheSentimentLocal2) { - int tokenform1size = cacheSentimentLocal1.getstrTokenForm().values().size(); - int tokenform2size = cacheSentimentLocal2.getstrTokenForm().values().size(); + private Double tokenformSizeScoring(Double score, ArrayList cacheSentimentLocal1, ArrayList cacheSentimentLocal2) { + int tokenform1size = cacheSentimentLocal1.size(); + int tokenform2size = cacheSentimentLocal2.size(); if (tokenform1size > 0 || tokenform2size > 0) { if (tokenform1size < tokenform2size * 5 && tokenform2size < tokenform1size * 5) { - for (String strTokenForm1itr1 : cacheSentimentLocal1.getstrTokenForm().values()) { - for (String strTokenForm1itr2 : cacheSentimentLocal2.getstrTokenForm().values()) { + for (String strTokenForm1itr1 : cacheSentimentLocal1) { + for (String strTokenForm1itr2 : cacheSentimentLocal2) { if (strTokenForm1itr1.equals(strTokenForm1itr2)) { score -= 1600; } else { @@ -1031,25 +1088,24 @@ public class SentimentAnalyzerTest { return score; } - private Double tokenStemmingMapScoring(Double score, SentimentValueCache cacheSentimentLocal1, SentimentValueCache cacheSentimentLocal2) { - ConcurrentMap tokenStemmingMap = new MapMaker().concurrencyLevel(5).makeMap(); - for (String strTokenStem : cacheSentimentLocal1.getstrTokenStems().values()) { - for (String strTokenStem1 : cacheSentimentLocal2.getstrTokenStems().values()) { - if (strTokenStem.equals(strTokenStem1) && !tokenStemmingMap.values().contains(strTokenStem)) { + private Double tokenStemmingMapScoring(Double score, ArrayList cacheSentimentLocal1, ArrayList cacheSentimentLocal2) { + ArrayList tokenStemmingMap = new ArrayList(); + for (String strTokenStem : cacheSentimentLocal1) { + for (String strTokenStem1 : cacheSentimentLocal2) { + if (strTokenStem.equals(strTokenStem1) && !tokenStemmingMap.contains(strTokenStem)) { score += 500; - tokenStemmingMap.put(tokenStemmingMap.size() + 1, strTokenStem); + tokenStemmingMap.add(strTokenStem); } } } return score; } - private Double inflectedCounterScoring(Double score, SentimentValueCache cacheSentimentLocal1, SentimentValueCache cacheSentimentLocal2) { - int inflectedCounterPositive1 = cacheSentimentLocal1.getInflectedCounterPositive(); - int inflectedCounterPositive2 = cacheSentimentLocal2.getInflectedCounterPositive(); - int inflectedCounterNegative = cacheSentimentLocal1.getInflectedCounterNegative() > cacheSentimentLocal2.getInflectedCounterNegative() - ? cacheSentimentLocal1.getInflectedCounterNegative() - cacheSentimentLocal2.getInflectedCounterNegative() - : cacheSentimentLocal2.getInflectedCounterNegative() - cacheSentimentLocal1.getInflectedCounterNegative(); + private Double inflectedCounterScoring(Double score, int inflectedCounterPositive1, int inflectedCounterPositive2, + int inflectedCounterNegative1, int inflectedCounterNegative2) { + int inflectedCounterNegative = inflectedCounterNegative1 > inflectedCounterNegative1 + ? inflectedCounterNegative1 - inflectedCounterNegative2 + : inflectedCounterNegative2 - inflectedCounterNegative1; if ((inflectedCounterPositive1 + inflectedCounterPositive2) > inflectedCounterNegative && inflectedCounterNegative > 0) { score += ((inflectedCounterPositive1 + inflectedCounterPositive2) - inflectedCounterNegative) * 650; } @@ -1068,9 +1124,7 @@ public class SentimentAnalyzerTest { return score; } - private Double annotatorCountScoring(Double score, SentimentValueCache cacheSentimentLocal1, SentimentValueCache cacheSentimentLocal2) { - int anotatorcounter1 = cacheSentimentLocal1.getAnotatorcounter(); - int anotatorcounter2 = cacheSentimentLocal2.getAnotatorcounter(); + private Double annotatorCountScoring(Double score, int anotatorcounter1, int anotatorcounter2) { if (anotatorcounter1 > 1 && anotatorcounter2 > 1) { if (anotatorcounter1 * 2 > anotatorcounter2 && anotatorcounter2 * 2 > anotatorcounter1) { score += anotatorcounter1 > anotatorcounter2 ? (anotatorcounter1 - anotatorcounter2) * 700 @@ -1082,9 +1136,7 @@ public class SentimentAnalyzerTest { return score; } - private Double tokensCounterScoring(Double score, SentimentValueCache cacheSentimentLocal1, SentimentValueCache cacheSentimentLocal2) { - int tokensCounter1 = cacheSentimentLocal1.getTokensCounter(); - int tokensCounter2 = cacheSentimentLocal2.getTokensCounter(); + private Double tokensCounterScoring(Double score, int tokensCounter1, int tokensCounter2) { if ((tokensCounter1 > 1 && tokensCounter2 > 1) && tokensCounter1 < tokensCounter2 * 5 && tokensCounter2 < tokensCounter1 * 5) { if (tokensCounter1 > tokensCounter2 / 2 && tokensCounter2 > tokensCounter1 / 2 && tokensCounter1 < 10 && tokensCounter2 < 10) { score += (tokensCounter1 + tokensCounter2) * 500; @@ -1113,36 +1165,12 @@ public class SentimentAnalyzerTest { return score; } - private SentimentValueCache setupNEREntitiesAndTokenTags(CoreDocument pipelineCoreDcoument, SentimentValueCache cacheSentimentLocal) { - for (CoreEntityMention em : pipelineCoreDcoument.entityMentions()) { - Set> entrySet = em.entityTypeConfidences().entrySet(); - String entityType = em.entityType(); - Double EntityConfidences = 0.0; - for (Map.Entry entries : entrySet) { - EntityConfidences = entries.getValue(); - } - List tokens = em.tokens(); - for (CoreLabel token : tokens) { - if (token != null) { - if (!cacheSentimentLocal.getnerEntityTokenTags().values().contains(token.tag())) { - if (entityType.equals("PERSON") && EntityConfidences > 0.80) { - cacheSentimentLocal.addnerEntityTokenTags(token.tag()); - } - } - } - } - if (!cacheSentimentLocal.getnerEntities1().values().contains(em.text())) { - cacheSentimentLocal.addNEREntities1(em.text()); - cacheSentimentLocal.addNEREntities2(em.entityType()); - } - } - return cacheSentimentLocal; - } - - private Double nerEntitiesAndTokenScoring(Double score, SentimentValueCache cacheSentimentLocal1, SentimentValueCache cacheSentimentLocal2) { + private Double nerEntitiesAndTokenScoring(Double score, ArrayList entityTokenTags1, + ArrayList entityTokenTags2, ArrayList nerEntities1, + ArrayList nerEntities2) { List entsCollection = new ArrayList(); - for (String strEnts1 : cacheSentimentLocal1.getnerEntities1().values()) { - for (String strEnts2 : cacheSentimentLocal2.getnerEntities2().values()) { + for (String strEnts1 : nerEntities1) { + for (String strEnts2 : nerEntities2) { if (strEnts1.equalsIgnoreCase(strEnts2) && !entsCollection.contains(strEnts1)) { score += 2500; entsCollection.add(strEnts1); @@ -1150,8 +1178,8 @@ public class SentimentAnalyzerTest { } } entsCollection = new ArrayList(); - for (String strToken : cacheSentimentLocal1.getnerEntityTokenTags().values()) { - for (String strToken1 : cacheSentimentLocal2.getnerEntityTokenTags().values()) { + for (String strToken : entityTokenTags1) { + for (String strToken1 : entityTokenTags2) { if (strToken.equalsIgnoreCase(strToken1) && !entsCollection.contains(strToken)) { score += 2000; entsCollection.add(strToken); @@ -1161,32 +1189,12 @@ public class SentimentAnalyzerTest { return score; } - private SentimentValueCache setupStoWordTokensLemma(Annotation pipelineAnnotationSentiment, SentimentValueCache cacheSentimentLocal) { - String customStopWordList = "start,starts,period,periods,a,an,and,are,as,at,be,but,by,for,if,in,into,is,it,no,not,of,on,or,such,that,the,their,then,there,these,they,this,to,was,will,with"; - List tokensSentiment = pipelineAnnotationSentiment.get(CoreAnnotations.TokensAnnotation.class); - Set stopWords = StopAnalyzer.ENGLISH_STOP_WORDS_SET; - Set stopWordsCustom = StopwordAnnotator.getStopWordList(customStopWordList, true); - for (CoreLabel token : tokensSentiment) { - Pair stopword = token.get(StopwordAnnotator.class); - String word = token.word().toLowerCase(); - if (stopWords.contains(word) || stopWordsCustom.contains(word)) { - cacheSentimentLocal.addstopwordTokens(word); - } - String lemma = token.lemma().toLowerCase(); - if (stopWords.contains(lemma) || stopWordsCustom.contains(lemma)) { - cacheSentimentLocal.addStopWordLemma(lemma); - } - if (stopword.first() && stopword.second()) { - cacheSentimentLocal.setPairCounter(cacheSentimentLocal.getPairCounter() + 1); - } - } - return cacheSentimentLocal; - } - - private Double stopWordTokenLemmaScoring(Double score, SentimentValueCache cacheSentimentLocal1, SentimentValueCache cacheSentimentLocal2) { + private Double stopWordTokenLemmaScoring(Double score, ArrayList stopWordToken1, + ArrayList stopWordToken2, ArrayList stopWordLemma1, + ArrayList stopWordLemma2) { Collection stopWordCollection = new ArrayList(); - for (String stopwords1 : cacheSentimentLocal1.getStopwordTokens().values()) { - for (String stopwords2 : cacheSentimentLocal2.getStopwordTokens().values()) { + for (String stopwords1 : stopWordToken1) { + for (String stopwords2 : stopWordToken2) { if (stopwords1.equals(stopwords2) && !stopWordCollection.contains(stopwords1)) { score -= 50; stopWordCollection.add(stopwords1); @@ -1194,8 +1202,8 @@ public class SentimentAnalyzerTest { } } stopWordCollection = new ArrayList(); - for (String stopwords1 : cacheSentimentLocal1.getStopWordLemma().values()) { - for (String stopwords2 : cacheSentimentLocal2.getStopWordLemma().values()) { + for (String stopwords1 : stopWordLemma1) { + for (String stopwords2 : stopWordLemma2) { if (stopwords1.equals(stopwords2) && !stopWordCollection.contains(stopwords1)) { score -= 50; stopWordCollection.add(stopwords1); @@ -1205,10 +1213,12 @@ public class SentimentAnalyzerTest { return score; } - private Double stopwordTokenPairCounterScoring(Double score, SentimentValueCache cacheSentimentLocal1, SentimentValueCache cacheSentimentLocal2) { - if (!cacheSentimentLocal1.getStopwordTokens().values().isEmpty() && !cacheSentimentLocal2.getStopwordTokens().values().isEmpty()) { - int stopwordsize1 = cacheSentimentLocal1.getStopwordTokens().values().size(); - int stopwordsize2 = cacheSentimentLocal2.getStopwordTokens().values().size(); + private Double stopwordTokenPairCounterScoring(Double score, ArrayList stopWordToken1, + ArrayList stopWordToken2, int pairCounter1, + int pairCounter2) { + if (!stopWordToken1.isEmpty() && !stopWordToken1.isEmpty()) { + int stopwordsize1 = stopWordToken1.size(); + int stopwordsize2 = stopWordToken1.size(); if (stopwordsize1 * 5 < stopwordsize2 || stopwordsize2 * 5 < stopwordsize1) { score -= stopwordsize1 > stopwordsize2 ? (stopwordsize1 - stopwordsize2) * 850 : (stopwordsize2 - stopwordsize1) * 850; } else if (stopwordsize1 == stopwordsize2 && stopwordsize1 > 1) { @@ -1219,8 +1229,6 @@ public class SentimentAnalyzerTest { score += stopwordsize1 > stopwordsize2 ? (stopwordsize1 - stopwordsize2) * 850 : (stopwordsize2 - stopwordsize1) * 850; } } - int pairCounter1 = cacheSentimentLocal1.getPairCounter(); - int pairCounter2 = cacheSentimentLocal2.getPairCounter(); if (pairCounter1 > 0 && pairCounter2 > 0) { if (pairCounter1 * 5 <= pairCounter2 || pairCounter2 * 5 <= pairCounter1) { score -= pairCounter1 > pairCounter2 ? (pairCounter1 - pairCounter2) * 1500 : (pairCounter2 - pairCounter1) * 1500; @@ -1235,242 +1243,354 @@ public class SentimentAnalyzerTest { return score; } - private Double tgwListScoreIncrementer(Double score, SentimentValueCache cacheSentimentLocal1, SentimentValueCache cacheSentimentLocal2) { - AtomicInteger runCount = new AtomicInteger(0); - for (String taggedWord : cacheSentimentLocal1.getTgwlistIndex().values()) { + private Double tgwListScoreIncrementer(Double score, ArrayList tgwListIndex1, + ArrayList tgwListIndex2) { + int runCount = 0; + for (String taggedWord : tgwListIndex1) { boolean found = false; - for (String taggedWord1 : cacheSentimentLocal2.getTgwlistIndex().values()) { + for (String taggedWord1 : tgwListIndex2) { if (taggedWord.equals(taggedWord1)) { found = true; break; } } if (!found) { - cacheSentimentLocal2.addTgwlistIndex(taggedWord); - runCount.getAndIncrement(); + runCount++; } } - score += runCount.get() * 64; + score += runCount * 64; return score; } + public void validateStringCaches() { + if (this.tokenizeCountingF == null) { + this.tokenizeCountingF = tokenizeCounting(getTaggedWordList(str1)); + } + if (this.tokenizeCounting == null) { + this.tokenizeCounting = tokenizeCounting(getTaggedWordList(str)); + } + if (this.taggedWordListF == null) { + this.taggedWordListF = getTaggedWordList(str); + } + if (this.taggedWordList1 == null) { + this.taggedWordList1 = getTaggedWordList(str1); + } + if (this.retrieveTGWListF == null) { + this.retrieveTGWListF = retrieveTGWListIndex(this.taggedWordListF); + } + if (this.retrieveTGWList1 == null) { + this.retrieveTGWList1 = retrieveTGWListIndex(this.taggedWordList1); + } + if (this.sentencesF == null) { + this.sentencesF = pipelineAnnotation1.get(CoreAnnotations.SentencesAnnotation.class); + } + if (this.sentences1 == null) { + this.sentences1 = pipelineAnnotation2.get(CoreAnnotations.SentencesAnnotation.class); + } + if (this.sentencesSentimentF == null) { + this.sentencesSentimentF = pipelineAnnotation1Sentiment.get(CoreAnnotations.SentencesAnnotation.class); + } + if (this.sentencesSentiment1 == null) { + this.sentencesSentiment1 = pipelineAnnotation2Sentiment.get(CoreAnnotations.SentencesAnnotation.class); + } + if (this.treesF == null) { + this.treesF = retrieveTrees(this.sentencesF); + } + if (this.trees1 == null) { + this.trees1 = retrieveTrees(this.sentences1); + } + } - public final SimilarityMatrix callSMX() { + + public SimilarityMatrix callSMX() { Double score = -100.0; - SentimentValueCache cacheSentimentLocal1 = null; - SentimentValueCache cacheSentimentLocal2 = null; int counter1; int counter2; - try { - if (cacheSentiment1 == null) { - cacheSentimentLocal1 = initializeCacheSetup(str, cacheSentimentLocal1); - } - if (cacheSentiment2 == null) { - cacheSentimentLocal2 = initializeCacheSetup(str1, cacheSentimentLocal2); - } - } catch (Exception ex) { - System.out.println("ex 1: " + ex.getMessage() + " " + ex.getLocalizedMessage() + " " - + ex.getCause()); - } - try { - counter1 = cacheSentiment1 == null ? cacheSentimentLocal1.getCounter() : cacheSentiment1.getCounter(); - counter2 = cacheSentiment2 == null ? cacheSentimentLocal2.getCounter() : cacheSentiment2.getCounter(); + validateStringCaches(); + counter1 = this.tokenizeCountingF; + counter2 = this.tokenizeCounting; + final int overValue = (counter1 >= counter2 ? counter1 - counter2 : counter2 - counter1) * 32; + score -= overValue; + score = tgwListScoreIncrementer(score, this.retrieveTGWListF, this.retrieveTGWList1); + Class sentimentAnnotatedTreeClass = + SentimentCoreAnnotations.SentimentAnnotatedTree.class; - final int overValue = (counter1 >= counter2 ? counter1 - counter2 : counter2 - counter1) * 32; - score -= overValue; - } catch (Exception ex) { - System.out.println("ex 2: " + ex.getMessage() + " " + ex.getLocalizedMessage() + " " - + ex.getCause()); - } - try { - if (cacheSentiment1 == null) { - ConcurrentMap retrieveTGWListIndex = retrieveTGWListIndex(cacheSentimentLocal1.getTaggedwordlist()); - for (String str : retrieveTGWListIndex.values()) { - cacheSentimentLocal1.addTgwlistIndex(str); + ArrayList grammaticalStructures1 = grammaticalStructureSetup(this.treesF); + ArrayList grammaticalStructures2 = grammaticalStructureSetup(this.trees1); + ArrayList typedDependencies1 = grammaticalStructureAllTypedDependencies(grammaticalStructures1); + ArrayList typedDependencies2 = grammaticalStructureAllTypedDependencies(grammaticalStructures2); + score = iterateTrees(this.treesF, this.trees1, score); + score = typeDependenciesGrammaticalRelation(typedDependencies1, typedDependencies2, score, grammaticalStructures1, grammaticalStructures2, + this.treesF, this.trees1); + ArrayList rnnCoreAnnotationsPredicted1 = sentimentRNNCoreAnnotations(this.sentencesSentimentF, sentimentAnnotatedTreeClass); + ArrayList rnnCoreAnnotationsPredicted2 = sentimentRNNCoreAnnotations(this.sentencesSentiment1, sentimentAnnotatedTreeClass); + ArrayList simpleMatrices1 = sentimentRNNCorePredicted(this.sentencesSentimentF, sentimentAnnotatedTreeClass); + ArrayList simpleMatrices2 = sentimentRNNCorePredicted(this.sentencesSentiment1, sentimentAnnotatedTreeClass); + ArrayList simpleMatricesNodevectors1 = sentimentRNNCoreNodevectors(this.sentencesSentimentF, sentimentAnnotatedTreeClass); + ArrayList simpleMatricesNodevectors2 = sentimentRNNCoreNodevectors(this.sentencesSentiment1, sentimentAnnotatedTreeClass); + score = simpleRNNMatrixCalculations(score, simpleMatrices1, simpleMatrices2); + score = simpleRNNMaxtrixVectors(score, simpleMatricesNodevectors1, simpleMatricesNodevectors2); + Integer sentiment1 = rnnCoreAnnotationsPredicted1.size(); + Integer sentiment2 = rnnCoreAnnotationsPredicted2.size(); + score -= (sentiment1 > sentiment2 ? sentiment2 - sentiment1 : sentiment1 - sentiment2) * 500; + score -= classifyRawEvaluation(); + + int longest1 = setupMainLongest(this.sentencesSentimentF); + int longest2 = setupMainLongest(this.sentencesSentiment1); + int sentimentLongest1 = setupMainSentiment(this.sentencesSentimentF, sentimentAnnotatedTreeClass); + int sentimentLongest2 = setupMainSentiment(this.sentencesSentiment1, sentimentAnnotatedTreeClass); + score = sentimentMatrixVariances(score, longest1, longest2, sentimentLongest1, sentimentLongest2); + + int InflectedCounterNegative1 = 0; + int InflectedCounterNegative2 = 0; + int InflectedCounterPositive1 = 0; + int InflectedCounterPositive2 = 0; + + ArrayList> coreMapsList = new ArrayList<>(); + coreMapsList.add(this.coreMaps1); + coreMapsList.add(this.coreMaps2); + + int MarkedContinuousCounter1 = 0; + int MarkedContinuousCounter2 = 0; + int UnmarkedPatternCounter1 = 0; + int UnmarkedPatternCounter2 = 0; + ArrayList strTokensIpartForm1 = new ArrayList<>(); + ArrayList strTokensIpartForm2 = new ArrayList<>(); + ArrayList tokenEntry1 = new ArrayList<>(); + ArrayList tokenEntry2 = new ArrayList<>(); + ArrayList tokenForms1 = new ArrayList<>(); + ArrayList tokenForms2 = new ArrayList<>(); + ArrayList strTokenEntryGetPOS1 = new ArrayList<>(); + ArrayList strTokenEntryGetPOS2 = new ArrayList<>(); + ArrayList intTokenEntyCounts1 = new ArrayList(); + ArrayList intTokenEntyCounts2 = new ArrayList(); + ArrayList ITokenTags1 = new ArrayList<>(); + ArrayList ITokenTags2 = new ArrayList<>(); + ArrayList strTokenStems1 = new ArrayList<>(); + ArrayList strTokenStems2 = new ArrayList<>(); + int Anotatorcounter1 = 0; + int Anotatorcounter2 = 0; + int TokensCounter1 = 0; + int TokensCounter2 = 0; + boolean firstAnnotation = true; + + for (List coreMaps : coreMapsList) { + int tokenadder = 0; + Collection> tokeninflectionMap = new ArrayList(); + for (CoreMap sentence : coreMaps) { + List> imwes = sentence.get(JMWEAnnotation.class); + for (IMWE token : imwes) { + Collection values = token.getPartMap().values(); + IMWEDesc entry = token.getEntry(); + if (!token.isInflected() && !tokeninflectionMap.contains(token)) { + if (firstAnnotation) { + InflectedCounterNegative1++; + } else { + InflectedCounterNegative2++; + } + tokeninflectionMap.add(token); + } else if (token.isInflected()) { + if (firstAnnotation) { + InflectedCounterPositive1++; + } else { + InflectedCounterPositive2++; + } + } + + String form = token.getForm(); + final String substring = token.getEntry().toString().substring(token.getEntry().toString().length() - 1); + if (firstAnnotation) { + tokenEntry1.add(substring); + tokenForms1.add(form); + } else { + tokenEntry2.add(substring); + tokenForms2.add(form); + } + if (firstAnnotation) { + MarkedContinuousCounter1 += entry.getMarkedContinuous(); + } else { + MarkedContinuousCounter2 += entry.getMarkedContinuous(); + } + if (firstAnnotation) { + UnmarkedPatternCounter1 += entry.getUnmarkedPattern(); + } else { + UnmarkedPatternCounter2 += entry.getUnmarkedPattern(); + } + for (IMWEDesc.IPart iPart : values) { + String iPartForm = iPart.getForm(); + if (firstAnnotation) { + strTokensIpartForm1.add(iPartForm); + } else { + strTokensIpartForm2.add(iPartForm); + } + } + for (String strPostPrefix : entry.getPOS().getPrefixes()) { + if (firstAnnotation) { + strTokenEntryGetPOS1.add(strPostPrefix); + } else { + strTokenEntryGetPOS2.add(strPostPrefix); + } + } + for (int counts : entry.getCounts()) { + if (firstAnnotation) { + intTokenEntyCounts1.add(counts); + } else { + intTokenEntyCounts2.add(counts); + } + } + for (IToken tokens : token.getTokens()) { + if (firstAnnotation) { + ITokenTags1.add(tokens.getTag()); + } else { + ITokenTags2.add(tokens.getTag()); + } + for (String strtoken : tokens.getStems()) { + if (firstAnnotation) { + strTokenStems1.add(strtoken); + } else { + strTokenStems2.add(strtoken); + } + if (firstAnnotation) { + MarkedContinuousCounter1++; + } else { + MarkedContinuousCounter2++; + } + } + } + tokenadder += 1; + } + if (firstAnnotation) { + Anotatorcounter1++; + } else { + Anotatorcounter2++; } } - if (cacheSentiment2 == null) { - ConcurrentMap retrieveTGWListIndex = retrieveTGWListIndex(cacheSentimentLocal2.getTaggedwordlist()); - for (String str : retrieveTGWListIndex.values()) { - cacheSentimentLocal2.addTgwlistIndex(str); - } + firstAnnotation = false; + if (firstAnnotation) { + TokensCounter1 = tokenadder; + } else { + TokensCounter2 = tokenadder; } - } catch (Exception ex) { - System.out.println("ex 3: " + ex.getMessage() + " " + ex.getLocalizedMessage() + " " - + ex.getCause()); - } - try { - score = tgwListScoreIncrementer(score, cacheSentiment1 == null - ? cacheSentimentLocal1 : cacheSentiment1, cacheSentiment2 == null ? cacheSentimentLocal2 : cacheSentiment2); - if (cacheSentiment1 == null) { - cacheSentimentLocal1 = GrammaticStructureSetup(cacheSentimentLocal1, pipelineAnnotation1); - } - if (cacheSentiment2 == null) { - cacheSentimentLocal2 = GrammaticStructureSetup(cacheSentimentLocal2, pipelineAnnotation2); - } - } catch (Exception ex) { - System.out.println("ex 4: " + ex.getMessage() + " " + ex.getLocalizedMessage() + " " - + ex.getCause()); - } - ConcurrentMap sentenceConstituencyParseList1 = null; - ConcurrentMap sentenceConstituencyParseList2 = null; - try { - sentenceConstituencyParseList2 = cacheSentiment2 == null - ? cacheSentimentLocal2.getSentenceConstituencyParseList() : cacheSentiment2.getSentenceConstituencyParseList(); - sentenceConstituencyParseList1 = cacheSentiment1 == null - ? cacheSentimentLocal1.getSentenceConstituencyParseList() : cacheSentiment1.getSentenceConstituencyParseList(); - score = iterateTrees(sentenceConstituencyParseList2, sentenceConstituencyParseList1, score); - } catch (Exception ex) { - System.out.println("ex 5: " + ex.getMessage() + " " + ex.getLocalizedMessage() + " " - + ex.getCause()); - } - try { - Collection allTypedDependencies2 = cacheSentiment2 == null ? cacheSentimentLocal2.getAllTypedDependencies() - : cacheSentiment2.getAllTypedDependencies(); - Collection allTypedDependencies1 = cacheSentiment1 == null ? cacheSentimentLocal1.getAllTypedDependencies() - : cacheSentiment1.getAllTypedDependencies(); - - ConcurrentMap grammaticalMap1 = cacheSentiment1 == null ? cacheSentimentLocal1.getGs() : cacheSentiment1.getGs(); - ConcurrentMap grammaticalMap2 = cacheSentiment2 == null ? cacheSentimentLocal2.getGs() : cacheSentiment2.getGs(); - score = typeDependenciesGrammaticalRelation(allTypedDependencies1, allTypedDependencies2, score, grammaticalMap1, grammaticalMap2, - sentenceConstituencyParseList1, sentenceConstituencyParseList2); - } catch (Exception ex) { - System.out.println("ex 6: " + ex.getMessage() + " " + ex.getLocalizedMessage() + " " - + ex.getCause()); - } - try { - if (cacheSentiment1 == null) { - cacheSentimentLocal1 = sentimentCoreAnnotationSetup(pipelineAnnotation1Sentiment, cacheSentimentLocal1); - } - if (cacheSentiment2 == null) { - cacheSentimentLocal2 = sentimentCoreAnnotationSetup(pipelineAnnotation2Sentiment, cacheSentimentLocal2); - } - } catch (Exception ex) { - System.out.println("ex 7: " + ex.getMessage() + " " + ex.getLocalizedMessage() + " " - + ex.getCause()); - } - try { - final ConcurrentMap simpleSMXlist1 = cacheSentiment1 == null - ? cacheSentimentLocal1.getSimpleSMXlist() : cacheSentiment1.getSimpleSMXlist(); - final ConcurrentMap simpleSMXlist2 = cacheSentiment2 == null - ? cacheSentimentLocal2.getSimpleSMXlist() : cacheSentiment2.getSimpleSMXlist(); - final ConcurrentMap simpleSMXlistVector1 = cacheSentiment1 == null - ? cacheSentimentLocal1.getSimpleSMXlistVector() : cacheSentiment1.getSimpleSMXlistVector(); - final ConcurrentMap simpleSMXlistVector2 = cacheSentiment2 == null - ? cacheSentimentLocal2.getSimpleSMXlistVector() : cacheSentiment2.getSimpleSMXlistVector(); - score = simpleRNNMatrixCalculations(score, simpleSMXlist1, simpleSMXlist2); - score = simpleRNNMaxtrixVectors(score, simpleSMXlistVector1, simpleSMXlistVector2); - } catch (Exception ex) { - System.out.println("ex 8: " + ex.getMessage() + " " + ex.getLocalizedMessage() + " " - + ex.getCause()); - } - try { - int sentiment1 = cacheSentiment1 == null ? cacheSentimentLocal1.getRnnPrediectClassMap().size() : cacheSentiment1.getRnnPrediectClassMap().size(); - int sentiment2 = cacheSentiment2 == null ? cacheSentimentLocal2.getRnnPrediectClassMap().size() : cacheSentiment2.getRnnPrediectClassMap().size(); - score -= (sentiment1 > sentiment2 ? sentiment1 - sentiment2 : sentiment2 - sentiment1) * 500; - Map.Entry> classifyRawEvaluationEntry = classifyRawEvaluation(score, cacheSentimentLocal1, - cacheSentimentLocal2); - score = classifyRawEvaluationEntry.getKey(); - if (cacheSentiment1 == null) { - cacheSentimentLocal1 = classifyRawEvaluationEntry.getValue().getKey(); - } - if (cacheSentiment2 == null) { - cacheSentimentLocal2 = classifyRawEvaluationEntry.getValue().getValue(); - } - } catch (Exception ex) { - System.out.println("ex 9: " + ex.getMessage() + " " + ex.getLocalizedMessage() + " " - + ex.getCause()); - } - try { - if (cacheSentiment1 == null) { - cacheSentimentLocal1 = setupMainSentimentandLongestVal(pipelineAnnotation1Sentiment, cacheSentimentLocal1); - } - if (cacheSentiment2 == null) { - cacheSentimentLocal2 = setupMainSentimentandLongestVal(pipelineAnnotation2Sentiment, cacheSentimentLocal2); - } - score = sentimentMatrixVariances(score, cacheSentiment1 == null ? cacheSentimentLocal1.getLongest() : cacheSentiment1.getLongest(), - cacheSentiment2 == null ? cacheSentimentLocal2.getLongest() : cacheSentiment2.getLongest(), cacheSentiment1 == null - ? cacheSentimentLocal1.getMainSentiment() : cacheSentiment1.getMainSentiment(), cacheSentiment2 == null - ? cacheSentimentLocal2.getMainSentiment() : cacheSentiment2.getMainSentiment()); - } catch (Exception ex) { - System.out.println("ex 10: " + ex.getMessage() + " " + ex.getLocalizedMessage() + " " - + ex.getCause()); - } - if (cacheSentiment1 == null) { - cacheSentimentLocal1 = jmweAnnotationSetup(jmweStrAnnotation1, cacheSentimentLocal1); - } - if (cacheSentiment2 == null) { - cacheSentimentLocal2 = jmweAnnotationSetup(jmweStrAnnotation2, cacheSentimentLocal2); } + score = entryCountsRelation(score, intTokenEntyCounts1, intTokenEntyCounts2); + score = entryCountsScoring(score, intTokenEntyCounts1, intTokenEntyCounts2); + score = tokenEntryPosScoring(score, strTokenEntryGetPOS1, strTokenEntryGetPOS2); + score = unmarkedPatternCounterScoring(score, UnmarkedPatternCounter1, UnmarkedPatternCounter2); + score = markedContiniousCounterScoring(score, MarkedContinuousCounter1, MarkedContinuousCounter2); + score = strTokensMapScoring(score, strTokensIpartForm1, strTokensIpartForm2); + score = strTokenEntryScoring(score, tokenEntry1, tokenEntry2); + score = strTokenMapTagsScoring(score, ITokenTags1, ITokenTags2); + score = tokenformSizeScoring(score, tokenForms1, tokenForms1); + score = tokenStemmingMapScoring(score, strTokenStems1, strTokenStems2); - SentimentValueCache scoringCache1 = cacheSentiment1 == null ? cacheSentimentLocal1 : cacheSentiment1; - SentimentValueCache scoringCache2 = cacheSentiment2 == null ? cacheSentimentLocal2 : cacheSentiment2; - score = entryCountsRelation(score, scoringCache1, scoringCache2); - score = entryCountsScoring(score, scoringCache1, scoringCache2); - - score = tokenEntryPosScoring(score, scoringCache1, scoringCache2); - - score = unmarkedPatternCounterScoring(score, scoringCache1, scoringCache2); - - score = markedContiniousCounterScoring(score, scoringCache1, scoringCache2); - - score = strTokensMapScoring(score, scoringCache1, scoringCache2); - - score = strTokenEntryScoring(score, scoringCache1, scoringCache2); - - score = strTokenMapTagsScoring(score, scoringCache1, scoringCache2); - - score = tokenformSizeScoring(score, scoringCache1, scoringCache2); - - score = tokenStemmingMapScoring(score, scoringCache1, scoringCache2); - - score = inflectedCounterScoring(score, scoringCache1, scoringCache2); - - score = annotatorCountScoring(score, scoringCache1, scoringCache2); - - score = tokensCounterScoring(score, scoringCache1, scoringCache2); - + score = inflectedCounterScoring(score, InflectedCounterPositive1, InflectedCounterPositive2, + InflectedCounterNegative1, InflectedCounterNegative2); + score = annotatorCountScoring(score, Anotatorcounter1, Anotatorcounter2); + score = tokensCounterScoring(score, TokensCounter1, TokensCounter2); LevenshteinDistance leven = new LevenshteinDistance(str, str1); double SentenceScoreDiff = leven.computeLevenshteinDistance(); SentenceScoreDiff *= 15; score -= SentenceScoreDiff; - try { - if (cacheSentiment1 == null) { - cacheSentimentLocal1 = setupNEREntitiesAndTokenTags(pipelineCoreDcoument1, cacheSentimentLocal1); + ArrayList entityTokenTags1 = new ArrayList<>(); + ArrayList entityTokenTags2 = new ArrayList<>(); + ArrayList nerEntities1 = new ArrayList<>(); + ArrayList nerEntities2 = new ArrayList<>(); + ArrayList nerEntitiesType = new ArrayList<>(); + ArrayList pipelineCoreDocuments = new ArrayList<>(); + pipelineCoreDocuments.add(pipelineCoreDcoument1); + pipelineCoreDocuments.add(pipelineCoreDcoument2); + firstAnnotation = true; + for (CoreDocument coreDocument : pipelineCoreDocuments) { + for (CoreEntityMention em : coreDocument.entityMentions()) { + Set> entrySet = em.entityTypeConfidences().entrySet(); + String entityType = em.entityType(); + Double EntityConfidences = 0.0; + for (Map.Entry entries : entrySet) { + EntityConfidences = entries.getValue(); + } + List tokens = em.tokens(); + for (CoreLabel token : tokens) { + if (token != null) { + if (firstAnnotation) { + if (!entityTokenTags1.contains(token.tag())) { + if (entityType.equals("PERSON") && EntityConfidences > 0.80) { + entityTokenTags1.add(token.tag()); + } + } + } else { + if (!entityTokenTags2.contains(token.tag())) { + if (entityType.equals("PERSON") && EntityConfidences > 0.80) { + entityTokenTags2.add(token.tag()); + } + } + } + } + } + if (firstAnnotation) { + if (!nerEntities1.contains(em.text())) { + nerEntities1.add(em.text()); + nerEntitiesType.add(em.entityType()); + } + } else { + if (!nerEntities2.contains(em.text())) { + nerEntities2.add(em.text()); + nerEntitiesType.add(em.entityType()); + } + } } - if (cacheSentiment2 == null) { - cacheSentimentLocal2 = setupNEREntitiesAndTokenTags(pipelineCoreDcoument2, cacheSentimentLocal2); - } - score = nerEntitiesAndTokenScoring(score, cacheSentiment1 == null ? cacheSentimentLocal1 : cacheSentiment1, cacheSentiment2 == null - ? cacheSentimentLocal2 : cacheSentiment2); - } catch (Exception ex) { - System.out.println("ex 11: " + ex.getMessage() + " " + ex.getLocalizedMessage() + " " - + ex.getCause()); + firstAnnotation = false; } - try { - if (cacheSentiment1 == null) { - cacheSentimentLocal1 = setupStoWordTokensLemma(pipelineAnnotation1Sentiment, cacheSentimentLocal1); + score = nerEntitiesAndTokenScoring(score, entityTokenTags1, entityTokenTags2, nerEntities1, nerEntities2); + ArrayList pipelineAnnotationSentiments = new ArrayList<>(); + ArrayList stopWordToken1 = new ArrayList<>(); + ArrayList stopWordToken2 = new ArrayList<>(); + ArrayList stopWordLemma1 = new ArrayList<>(); + ArrayList stopWordLemma2 = new ArrayList<>(); + int PairCounter1 = 0; + int PairCounter2 = 0; + pipelineAnnotationSentiments.add(pipelineAnnotation1Sentiment); + pipelineAnnotationSentiments.add(pipelineAnnotation2Sentiment); + + firstAnnotation = true; + for (Annotation pipelineAnnotationSentiment : pipelineAnnotationSentiments) { + String customStopWordList = "start,starts,period,periods,a,an,and,are,as,at,be,but,by,for,if,in," + + "into,is,it,no,not,of,on,or,such,that,the,their,then,there,these,they,this,to,was,will,with"; + List tokensSentiment = pipelineAnnotationSentiment.get(CoreAnnotations.TokensAnnotation.class); + Set stopWords = StopAnalyzer.ENGLISH_STOP_WORDS_SET; + Set stopWordsCustom = StopwordAnnotator.getStopWordList(customStopWordList, true); + for (CoreLabel token : tokensSentiment) { + Pair stopword = token.get(StopwordAnnotator.class); + String word = token.word().toLowerCase(); + if (stopWords.contains(word) || stopWordsCustom.contains(word)) { + if (firstAnnotation) { + stopWordToken1.add(word); + } else { + stopWordToken2.add(word); + } + } + String lemma = token.lemma().toLowerCase(); + if (stopWords.contains(lemma) || stopWordsCustom.contains(lemma)) { + if (firstAnnotation) { + stopWordLemma1.add(lemma); + } else { + stopWordLemma2.add(lemma); + } + } + if (stopword.first() && stopword.second()) { + if (firstAnnotation) { + PairCounter1++; + } else { + PairCounter2++; + } + } } - if (cacheSentiment2 == null) { - cacheSentimentLocal2 = setupStoWordTokensLemma(pipelineAnnotation2Sentiment, cacheSentimentLocal2); - } - score = stopWordTokenLemmaScoring(score, cacheSentiment1 == null ? cacheSentimentLocal1 : cacheSentiment1, cacheSentiment2 == null - ? cacheSentimentLocal2 : cacheSentiment2); - } catch (Exception ex) { - System.out.println("ex 12: " + ex.getMessage() + " " + ex.getLocalizedMessage() + " " - + ex.getCause()); + firstAnnotation = false; } - score = stopwordTokenPairCounterScoring(score, cacheSentiment1 == null ? cacheSentimentLocal1 : cacheSentiment1, cacheSentiment2 == null - ? cacheSentimentLocal2 : cacheSentiment2); + score = stopWordTokenLemmaScoring(score, stopWordToken1, stopWordToken2, stopWordLemma1, stopWordLemma2); + score = stopwordTokenPairCounterScoring(score, stopWordToken1, stopWordToken2, PairCounter1, PairCounter2); smxParam.setDistance(score); - if (cacheSentiment1 == null) { - smxParam.setCacheValue1(cacheSentimentLocal1); - } - if (cacheSentiment2 == null) { - smxParam.setCacheValue2(cacheSentimentLocal2); - } return smxParam; } } diff --git a/ArtificialAutism/src/main/java/FunctionLayer/StanfordParser/SentimentValueCache.java b/ArtificialAutism/src/main/java/FunctionLayer/StanfordParser/SentimentValueCache.java deleted file mode 100644 index 33455779..00000000 --- a/ArtificialAutism/src/main/java/FunctionLayer/StanfordParser/SentimentValueCache.java +++ /dev/null @@ -1,334 +0,0 @@ -/* - * To change this license header, choose License Headers in Project Properties. - * To change this template file, choose Tools | Templates - * and open the template in the editor. - */ -package FunctionLayer.StanfordParser; - -import com.google.common.collect.MapMaker; -import edu.stanford.nlp.ling.TaggedWord; -import edu.stanford.nlp.trees.GrammaticalStructure; -import edu.stanford.nlp.trees.Tree; -import edu.stanford.nlp.trees.TypedDependency; -import java.util.ArrayList; -import java.util.Collection; -import java.util.List; -import java.util.Map; -import java.util.concurrent.ConcurrentMap; -import org.ejml.simple.SimpleMatrix; - -/** - * - * @author install1 - */ -public class SentimentValueCache { - - private String sentence; - private int counter; - private List> taggedwordlist = new ArrayList(); - private final ConcurrentMap tgwlistIndex = new MapMaker().concurrencyLevel(2).makeMap(); - private final ConcurrentMap sentenceConstituencyParseList = new MapMaker().concurrencyLevel(2).makeMap(); - private final Collection allTypedDependencies = new ArrayList(); - private final ConcurrentMap gsMap = new MapMaker().concurrencyLevel(2).makeMap(); - private final ConcurrentMap simpleSMXlist = new MapMaker().concurrencyLevel(3).makeMap(); - private final ConcurrentMap simpleSMXlistVector = new MapMaker().concurrencyLevel(3).makeMap(); - private final ConcurrentMap rnnPredictClassMap = new MapMaker().concurrencyLevel(3).makeMap(); - private List classifyRaw; - private int mainSentiment = 0; - private int longest = 0; - private int tokensCounter = 0; - private int anotatorcounter = 0; - private int inflectedCounterPositive = 0; - private int inflectedCounterNegative = 0; - private int MarkedContinuousCounter = 0; - private int MarkedContiniousCounterEntries = 0; - private int UnmarkedPatternCounter = 0; - private int pairCounter = 0; - private final ConcurrentMap ITokenMapTag = new MapMaker().concurrencyLevel(2).makeMap(); - private final ConcurrentMap strTokenStems = new MapMaker().concurrencyLevel(2).makeMap(); - private final ConcurrentMap strTokenForm = new MapMaker().concurrencyLevel(2).makeMap(); - private final ConcurrentMap strTokenGetEntry = new MapMaker().concurrencyLevel(2).makeMap(); - private final ConcurrentMap strTokenGetiPart = new MapMaker().concurrencyLevel(2).makeMap(); - private final ConcurrentMap strTokenEntryPOS = new MapMaker().concurrencyLevel(2).makeMap(); - private final ConcurrentMap entryCounts = new MapMaker().concurrencyLevel(2).makeMap(); - private final ConcurrentMap nerEntities1 = new MapMaker().concurrencyLevel(2).makeMap(); - private final ConcurrentMap nerEntities2 = new MapMaker().concurrencyLevel(2).makeMap(); - private final ConcurrentMap nerEntityTokenTags = new MapMaker().concurrencyLevel(3).makeMap(); - private final ConcurrentMap stopwordTokens = new MapMaker().concurrencyLevel(2).makeMap(); - private final ConcurrentMap stopWordLemma = new MapMaker().concurrencyLevel(2).makeMap(); - - public int getPairCounter() { - return pairCounter; - } - - public void setPairCounter(int pairCounter) { - this.pairCounter = pairCounter; - } - - public void addStopWordLemma(String str) { - stopWordLemma.put(stopWordLemma.size(), str); - } - - public void addstopwordTokens(String str) { - stopwordTokens.put(stopwordTokens.size(), str); - } - - public ConcurrentMap getStopwordTokens() { - return stopwordTokens; - } - - public ConcurrentMap getStopWordLemma() { - return stopWordLemma; - } - - public void addnerEntityTokenTags(String str) { - nerEntityTokenTags.put(nerEntityTokenTags.size(), str); - } - - public ConcurrentMap getnerEntityTokenTags() { - return nerEntityTokenTags; - } - - public ConcurrentMap getnerEntities1() { - return nerEntities1; - } - - public ConcurrentMap getnerEntities2() { - return nerEntities2; - } - - public void addNEREntities1(String str) { - nerEntities1.put(nerEntities1.size(), str); - } - - public void addNEREntities2(String str) { - nerEntities2.put(nerEntities2.size(), str); - } - - public void setTaggedwords(List> twlist) { - taggedwordlist = twlist; - } - - public List> getTaggedwordlist() { - return taggedwordlist; - } - - public void addEntryCounts(int counts) { - entryCounts.put(entryCounts.size(), counts); - } - - public ConcurrentMap getEntryCounts() { - return entryCounts; - } - - public void addstrTokenEntryPOS(String str) { - strTokenEntryPOS.put(strTokenEntryPOS.size(), str); - } - - public ConcurrentMap getstrTokenEntryPOS() { - return strTokenEntryPOS; - } - - public void addstrTokenGetiPart(String str) { - strTokenGetiPart.put(strTokenGetiPart.size(), str); - } - - public ConcurrentMap getstrTokenGetiPart() { - return strTokenGetiPart; - } - - public ConcurrentMap getstrTokenGetEntry() { - return strTokenGetEntry; - } - - public void addstrTokenGetEntry(String str) { - strTokenGetEntry.put(strTokenGetEntry.size(), str); - } - - public ConcurrentMap getstrTokenForm() { - return strTokenForm; - } - - public void addstrTokenForm(String str) { - strTokenForm.put(strTokenForm.size(), str); - } - - public ConcurrentMap getstrTokenStems() { - return strTokenStems; - } - - public void addstrTokenStems(String str) { - strTokenStems.put(strTokenStems.size(), str); - } - - public ConcurrentMap getITokenMapTag() { - return ITokenMapTag; - } - - public void addITokenMapTag(String str) { - ITokenMapTag.put(ITokenMapTag.size(), str); - } - - public int getUnmarkedPatternCounter() { - return UnmarkedPatternCounter; - } - - public void setUnmarkedPatternCounter(int UnmarkedPatternCounter) { - this.UnmarkedPatternCounter = UnmarkedPatternCounter; - } - - public int getMarkedContiniousCounterEntries() { - return MarkedContiniousCounterEntries; - } - - public void setMarkedContiniousCounterEntries(int MarkedContiniousCounterEntries) { - this.MarkedContiniousCounterEntries = MarkedContiniousCounterEntries; - } - - public int getMarkedContinuousCounter() { - return MarkedContinuousCounter; - } - - public void setMarkedContinuousCounter(int MarkedContinuousCounter) { - this.MarkedContinuousCounter = MarkedContinuousCounter; - } - - public int getInflectedCounterNegative() { - return inflectedCounterNegative; - } - - public void setInflectedCounterNegative(int inflectedCounterNegative) { - this.inflectedCounterNegative = inflectedCounterNegative; - } - - public int getInflectedCounterPositive() { - return inflectedCounterPositive; - } - - public void setInflectedCounterPositive(int inflectedCounterPositive) { - this.inflectedCounterPositive = inflectedCounterPositive; - } - - public int getAnotatorcounter() { - return anotatorcounter; - } - - public void setAnotatorcounter(int anotatorcounter) { - this.anotatorcounter = anotatorcounter; - } - - public int getTokensCounter() { - return tokensCounter; - } - - public void setTokensCounter(int tokensCounter) { - this.tokensCounter = tokensCounter; - } - - public int getMainSentiment() { - return mainSentiment; - } - - public void setMainSentiment(int mainSentiment) { - this.mainSentiment = mainSentiment; - } - - public int getLongest() { - return longest; - } - - public void setLongest(int longest) { - this.longest = longest; - } - - public List getClassifyRaw() { - return classifyRaw; - } - - public void setClassifyRaw(List classifyRaw) { - this.classifyRaw = classifyRaw; - } - - public ConcurrentMap getRnnPrediectClassMap() { - return rnnPredictClassMap; - } - - public void addRNNPredictClass(int rnnPrediction) { - rnnPredictClassMap.put(rnnPredictClassMap.size(), rnnPrediction); - } - - public void addSimpleMatrix(SimpleMatrix SMX) { - simpleSMXlist.put(simpleSMXlist.size(), SMX); - } - - public void addSimpleMatrixVector(SimpleMatrix SMX) { - simpleSMXlistVector.put(simpleSMXlistVector.size(), SMX); - } - - public ConcurrentMap getGsMap() { - return gsMap; - } - - public ConcurrentMap getSimpleSMXlist() { - return simpleSMXlist; - } - - public ConcurrentMap getSimpleSMXlistVector() { - return simpleSMXlistVector; - } - - public ConcurrentMap getGs() { - return gsMap; - } - - public int getCounter() { - return counter; - } - - public void addGS(GrammaticalStructure gs) { - gsMap.put(gsMap.size(), gs); - } - - public Collection getAllTypedDependencies() { - return allTypedDependencies; - } - - public void addTypedDependencies(Collection TDPlist) { - for (TypedDependency TDP : TDPlist) { - allTypedDependencies.add(TDP); - } - } - - public ConcurrentMap getSentenceConstituencyParseList() { - return sentenceConstituencyParseList; - } - - public void addSentenceConstituencyParse(Tree tree) { - sentenceConstituencyParseList.put(sentenceConstituencyParseList.size(), tree); - } - - public void setCounter(int counter) { - counter = counter; - } - - public String getSentence() { - return sentence; - } - - public SentimentValueCache(String str, int counter) { - this.sentence = str; - this.counter = counter; - } - - public ConcurrentMap getTgwlistIndex() { - return tgwlistIndex; - } - - public void addTgwlistIndex(String str) { - tgwlistIndex.put(tgwlistIndex.size(), str); - } - - public SentimentValueCache(String str) { - this.sentence = str; - } -} diff --git a/ArtificialAutism/src/main/java/PresentationLayer/DiscordHandler.java b/ArtificialAutism/src/main/java/PresentationLayer/DiscordHandler.java index ea4d7766..a00c4c89 100644 --- a/ArtificialAutism/src/main/java/PresentationLayer/DiscordHandler.java +++ b/ArtificialAutism/src/main/java/PresentationLayer/DiscordHandler.java @@ -15,21 +15,18 @@ screen -X -S (number1) quit package PresentationLayer; import DataLayer.settings; -import FunctionLayer.CustomError; import FunctionLayer.Datahandler; import FunctionLayer.PipelineJMWESingleton; import com.sun.tools.javac.util.List; import discord4j.core.DiscordClient; import discord4j.core.GatewayDiscordClient; import discord4j.core.event.domain.message.MessageCreateEvent; +import edu.stanford.nlp.pipeline.StanfordCoreNLP; import java.io.IOException; import java.io.UnsupportedEncodingException; import java.net.*; import java.sql.SQLException; -import java.util.ArrayList; -import java.util.logging.Level; -import java.util.logging.Logger; /** @@ -37,7 +34,8 @@ import java.util.logging.Logger; */ public class DiscordHandler { - private static void receiveAndSendPacket(DatagramSocket serverSocket, InetAddress ipAddress, int port) throws IOException, CustomError { + private static void receiveAndSendPacket(DatagramSocket serverSocket, InetAddress ipAddress, int port, + Datahandler datahandler, StanfordCoreNLP stanfordCoreNLP, StanfordCoreNLP stanfordCoreNLPSentiment) throws IOException { byte[] receiveData = new byte[4096]; DatagramPacket receivePacket = new DatagramPacket(receiveData, receiveData.length); try { @@ -48,8 +46,9 @@ public class DiscordHandler { String sentence = new String(receivePacket.getData(), 0, receivePacket.getLength()); sentence = sentence.replace("clientmessage:", ""); - String getResponseMsg = Datahandler.instance.getResponseMsg(sentence); - byte[] sendData = getResponseMsg.getBytes("UTF-8"); + String ResponseMsg = datahandler.getResponseMsg(sentence, "", stanfordCoreNLP, stanfordCoreNLPSentiment, + true); + byte[] sendData = ResponseMsg.getBytes("UTF-8"); int deliver_port = 0; switch (port) { case 48470: @@ -67,10 +66,10 @@ public class DiscordHandler { } DatagramPacket sendPacket = new DatagramPacket(sendData, sendData.length, ipAddress, deliver_port); serverSocket.send(sendPacket); - //System.out.println("receiveAndSendPacket send message to port: " + deliver_port); } - public static void handleUDPTraffic(int port) { + public static void handleUDPTraffic(int port, Datahandler datahandler, + StanfordCoreNLP stanfordCoreNLP, StanfordCoreNLP stanfordCoreNLPSentiment) { try (DatagramSocket serverSocket = new DatagramSocket(port)) { String hostIP = ""; if (port == 48473 || port == 48471) { @@ -80,35 +79,27 @@ public class DiscordHandler { } InetAddress ipAddress = InetAddress.getByName(hostIP);//used ip' while (true) { - receiveAndSendPacket(serverSocket, ipAddress, port); + receiveAndSendPacket(serverSocket, ipAddress, port, datahandler, stanfordCoreNLP, stanfordCoreNLPSentiment); } } catch (SocketException | UnknownHostException e) { e.printStackTrace(); } catch (UnsupportedEncodingException e) { e.printStackTrace(); - } catch (CustomError customError) { - customError.printStackTrace(); } catch (IOException e) { e.printStackTrace(); } } - public static void main(String[] args) { - try { - Datahandler.instance.initiateMYSQL(); - //nohup screen -d -m -S nonroot java -Xmx6900M -jar /home/javatests/ArtificialAutism-1.0.jar - //uncomment db fetch when ready, just keep the comment for future reference - System.out.println("finished initiating MYSQL"); - } catch (SQLException | IOException ex) { - Logger.getLogger(DiscordHandler.class.getName()).log(Level.SEVERE, null, ex); - } + public static void main(String[] args) throws IOException, SQLException { + Datahandler datahandler = new Datahandler(); + datahandler.initiateMYSQL(); + PipelineJMWESingleton.getINSTANCE(); - Datahandler.instance.instantiateAnnotationMapJMWE(); - Datahandler.instance.shiftReduceParserInitiate(); - Datahandler.instance.instantiateAnnotationMap(); + StanfordCoreNLP stanfordCoreNLP = datahandler.pipeLineSetUp(); + StanfordCoreNLP stanfordCoreNLPSentiment = datahandler.shiftReduceParserInitiate(); System.out.println("FINISHED ALL ANNOTATIONS"); - Datahandler.instance.updateStringCache(); - //String token = "NTI5NzAxNTk5NjAyMjc4NDAx.Dw0vDg.7-aMjVWdQMYPl8qVNyvTCPS5F_A"; + datahandler.updateStringCache(); + System.out.println("updatedstring cache"); String token = new settings().getDiscordToken(); final DiscordClient client = DiscordClient.create(token); final GatewayDiscordClient gateway = client.login().block(); @@ -119,13 +110,11 @@ public class DiscordHandler { final int j = i; new Thread(() -> { List ports = List.of(48470, 48471, 48472, 48473); - handleUDPTraffic(ports.get(j)); + handleUDPTraffic(ports.get(j), datahandler, stanfordCoreNLP, stanfordCoreNLPSentiment); }).start(); } gateway.on(MessageCreateEvent.class).subscribe(event -> { - if (!FunctionLayer.DoStuff.isOccupied()) { - FunctionLayer.DoStuff.doStuff(event, usernameBot); - } + FunctionLayer.DoStuff.doStuff(event, usernameBot, datahandler, stanfordCoreNLP, stanfordCoreNLPSentiment); }); gateway.onDisconnect().block(); }