UNLOZE/projects-jenz
9
0
Fork 0
Code Issues Pull Requests Releases Wiki Activity

replacing most arraylists with concurrentmaps due to them requirring less bytes per header since they should be stored bucketwise

Browse Source
This commit is contained in:
jenzur
2019-03-03 23:32:36 +01:00
parent f64ce5c5a0
commit aff2ddae5b
5 changed files with 200 additions and 227 deletions
Download Patch File Download Diff File Expand all files Collapse all files
ArtificialAutism/src/main/java/DataLayer/DBCPDataSource.java
-1
View File
@@ -21,7 +21,6 @@ public class DBCPDataSource {
try {
ds.setDriver(new com.mysql.cj.jdbc.Driver());
ds.setUrl("jdbc:mysql://151.80.230.149:3306/ArtificialAutism?useLegacyDatetimeCode=false&serverTimezone=UTC");
//ds.setUrl("jdbc:mysql://localhost:3306/ArtificialAutism?useLegacyDatetimeCode=false&serverTimezone=UTC");
ds.setUsername("ArtificialAutism");
ds.setPassword("b423b54bwbfb1340438fn");
ds.setMaxTotal(-1);
ArtificialAutism/src/main/java/DataLayer/DataMapper.java
+13 -47
View File
@@ -17,6 +17,7 @@ import java.util.HashMap;
import java.util.LinkedHashMap;
import java.util.List;
import java.util.Map;
import java.util.concurrent.ConcurrentMap;
import java.util.logging.Level;
import java.util.logging.Logger;
@@ -72,24 +73,22 @@ public class DataMapper {
return str;
}
public static void InsertMYSQLStrings(List<String> str) throws CustomError {
public static void InsertMYSQLStrings(ConcurrentMap<Integer, String> str) throws CustomError {
Connection l_cCon = null;
PreparedStatement l_pStatement = null;
ResultSet l_rsSearch = null;
String l_sSQL = "INSERT IGNORE `Sentences` (`Strings`) VALUES (?)";
try {
if (str != null && str.size() > 0) {
l_cCon = DBCPDataSource.getConnection();
l_pStatement = l_cCon.prepareStatement(l_sSQL, java.sql.ResultSet.TYPE_FORWARD_ONLY,
java.sql.ResultSet.CONCUR_READ_ONLY);
l_pStatement.setFetchSize(Integer.MIN_VALUE);
for (String str1 : str) {
System.out.println("adding str1: " + str1 + "\n");
l_pStatement.setString(1, str1);
l_pStatement.addBatch();
}
l_pStatement.executeBatch();
l_cCon = DBCPDataSource.getConnection();
l_pStatement = l_cCon.prepareStatement(l_sSQL, java.sql.ResultSet.TYPE_FORWARD_ONLY,
java.sql.ResultSet.CONCUR_READ_ONLY);
l_pStatement.setFetchSize(Integer.MIN_VALUE);
for (String str1 : str.values()) {
System.out.println("adding str1: " + str1 + "\n");
l_pStatement.setString(1, str1);
l_pStatement.addBatch();
}
l_pStatement.executeBatch();
} catch (SQLException ex) {
throw new CustomError("failed in DataMapper " + ex.getMessage());
} finally {
@@ -111,40 +110,7 @@ public class DataMapper {
return count;
}
public static List<SimilarityMatrix> getAllSementicMatrixes() throws CustomError {
//https://stackoverflow.com/questions/5157476/resultset-behavior-with-mysql-database-does-it-store-all-rows-in-memory/5159999#5159999
//https://stackoverflow.com/questions/3682614/how-to-read-all-rows-from-huge-table
int count = getSementicsDBRows();
int counter2 = 0;
int hardCapRetrieveCount = 500000;
List<SimilarityMatrix> WS4JList = new ArrayList(count + 1);
while (count > counter2) {
try (Connection l_cCon = DBCPDataSource.getConnection()) {
l_cCon.setAutoCommit(false);
String l_sSQL = "SELECT * FROM `WordMatrix` WHERE ID > " + counter2 + " AND ID < " + (counter2 + hardCapRetrieveCount);
try (PreparedStatement l_pStatement = l_cCon.prepareStatement(l_sSQL, java.sql.ResultSet.TYPE_FORWARD_ONLY,
java.sql.ResultSet.CONCUR_READ_ONLY)) {
l_pStatement.setFetchSize(Integer.MIN_VALUE);
try (ResultSet l_rsSearch = l_pStatement.executeQuery()) {
int i = 0;
while (l_rsSearch.next() && i < hardCapRetrieveCount) {
SimilarityMatrix ws4j = new SimilarityMatrix(l_rsSearch.getString(1), l_rsSearch.getString(2), l_rsSearch.getDouble(3));
//find something cheaper than arraylist probably
WS4JList.add(ws4j);
System.out.println("i: " + i + "\n" + "free memory: " + Runtime.getRuntime().freeMemory() + "\ncounter2: " + counter2 + "\n");
i++;
counter2++;
}
}
}
} catch (SQLException ex) {
Logger.getLogger(DataMapper.class.getName()).log(Level.SEVERE, null, ex);
}
}
return WS4JList;
}
public static void insertSementicMatrixes(List<SimilarityMatrix> WS4JListUpdate) throws CustomError {
public static void insertSementicMatrixes(ConcurrentMap<Integer, SimilarityMatrix> WS4JListUpdate) throws CustomError {
Connection l_cCon = null;
PreparedStatement l_pStatement = null;
ResultSet l_rsSearch = null;
@@ -155,7 +121,7 @@ public class DataMapper {
java.sql.ResultSet.CONCUR_READ_ONLY);
l_pStatement.setFetchSize(Integer.MIN_VALUE);
System.out.println("Matrix update size: " + WS4JListUpdate.size());
for (SimilarityMatrix ws4j : WS4JListUpdate) {
for (SimilarityMatrix ws4j : WS4JListUpdate.values()) {
l_pStatement.setString(1, ws4j.getPrimaryString());
l_pStatement.setString(2, ws4j.getSecondaryString());
l_pStatement.setDouble(3, ws4j.getDistance());