diff --git a/.gitignore b/.gitignore index 9459f98..8d10479 100644 --- a/.gitignore +++ b/.gitignore @@ -14,3 +14,6 @@ clean.bash rds-truststore.jks log run.tmc.bash + +.idea + diff --git a/config.bash b/config.bash index f26e63d..90c9c20 100644 --- a/config.bash +++ b/config.bash @@ -1,5 +1,8 @@ # You will most likely want to change these values +# customize replica setname +export MONGODB_REPLICASETNAME=adt + export SHARDED=no # database in which to run the benchmark @@ -148,4 +151,3 @@ export MONGO_BASEMENT=65536 # set to N to use begin/commit/ensure for TokuMX # valid values : N or Y export SYSBENCH_AUTO_COMMIT=Y - diff --git a/run.simple.bash b/run.simple.bash index 8eb923c..f4653a1 100755 --- a/run.simple.bash +++ b/run.simple.bash @@ -47,10 +47,10 @@ if [[ $DOLOAD = "yes" ]]; then if [[ $SHARDED = "no" ]]; then echo "executing standard sysbench load (not sharded)" | tee -a $LOG_NAME - java -cp $CLASSPATH:$PWD/src jmongosysbenchload $NUM_COLLECTIONS $DB_NAME $NUM_LOADER_THREADS $NUM_DOCUMENTS_PER_COLLECTION $NUM_DOCUMENTS_PER_INSERT $NUM_INSERTS_PER_FEEDBACK $NUM_SECONDS_PER_FEEDBACK $BENCHMARK_TSV $MONGO_COMPRESSION $MONGO_BASEMENT $WRITE_CONCERN $MONGO_SERVER $MONGO_PORT "$USERNAME" "$PASSWORD" "$TRUST_STORE" "$TRUST_STORE_PASSWORD" $USE_TLS + java -cp $CLASSPATH:$PWD/src jmongosysbenchload $NUM_COLLECTIONS $DB_NAME $NUM_LOADER_THREADS $NUM_DOCUMENTS_PER_COLLECTION $NUM_DOCUMENTS_PER_INSERT $NUM_INSERTS_PER_FEEDBACK $NUM_SECONDS_PER_FEEDBACK $BENCHMARK_TSV $MONGO_COMPRESSION $MONGO_BASEMENT $WRITE_CONCERN $MONGO_SERVER $MONGO_PORT "$USERNAME" "$PASSWORD" "$TRUST_STORE" "$TRUST_STORE_PASSWORD" $USE_TLS "$MONGODB_REPLICASETNAME" else echo "executing sharded sysbench load" | tee -a $LOG_NAME - java -cp $CLASSPATH:$PWD/src jmongosysbenchshardedload $NUM_COLLECTIONS $DB_NAME $NUM_LOADER_THREADS $NUM_DOCUMENTS_PER_COLLECTION $NUM_DOCUMENTS_PER_INSERT $NUM_INSERTS_PER_FEEDBACK $NUM_SECONDS_PER_FEEDBACK $BENCHMARK_TSV $MONGO_COMPRESSION $MONGO_BASEMENT $WRITE_CONCERN $MONGO_SERVER $MONGO_PORT "$USERNAME" "$PASSWORD" "$TRUST_STORE" "$TRUST_STORE_PASSWORD" $USE_TLS $MAX_SHARD_KEY $DOCS_PER_SHARD + java -cp $CLASSPATH:$PWD/src jmongosysbenchshardedload $NUM_COLLECTIONS $DB_NAME $NUM_LOADER_THREADS $NUM_DOCUMENTS_PER_COLLECTION $NUM_DOCUMENTS_PER_INSERT $NUM_INSERTS_PER_FEEDBACK $NUM_SECONDS_PER_FEEDBACK $BENCHMARK_TSV $MONGO_COMPRESSION $MONGO_BASEMENT $WRITE_CONCERN $MONGO_SERVER $MONGO_PORT "$USERNAME" "$PASSWORD" "$TRUST_STORE" "$TRUST_STORE_PASSWORD" $USE_TLS $MAX_SHARD_KEY $DOCS_PER_SHARD "$MONGODB_REPLICASETNAME" fi echo "" | tee -a $LOG_NAME @@ -78,10 +78,10 @@ if [[ $DOQUERY = "yes" ]]; then if [[ $SHARDED = "no" ]]; then echo "executing standard sysbench" | tee -a $LOG_NAME - java -cp $CLASSPATH:$PWD/src jmongosysbenchexecute $NUM_COLLECTIONS $DB_NAME $NUM_WRITER_THREADS $NUM_DOCUMENTS_PER_COLLECTION $NUM_SECONDS_PER_FEEDBACK $BENCHMARK_TSV $SYSBENCH_AUTO_COMMIT $RUN_TIME_SECONDS $SYSBENCH_RANGE_SIZE $SYSBENCH_POINT_SELECTS $SYSBENCH_SIMPLE_RANGES $SYSBENCH_SUM_RANGES $SYSBENCH_ORDER_RANGES $SYSBENCH_DISTINCT_RANGES $SYSBENCH_INDEX_UPDATES $SYSBENCH_NON_INDEX_UPDATES $SYSBENCH_INSERTS $WRITE_CONCERN $MAX_TPS $MONGO_SERVER $MONGO_PORT $SEED "$USERNAME" "$PASSWORD" $MONGO_READ_PREFERENCE "$TRUST_STORE" "$TRUST_STORE_PASSWORD" $USE_TLS | tee -a $LOG_NAME + java -cp $CLASSPATH:$PWD/src jmongosysbenchexecute $NUM_COLLECTIONS $DB_NAME $NUM_WRITER_THREADS $NUM_DOCUMENTS_PER_COLLECTION $NUM_SECONDS_PER_FEEDBACK $BENCHMARK_TSV $SYSBENCH_AUTO_COMMIT $RUN_TIME_SECONDS $SYSBENCH_RANGE_SIZE $SYSBENCH_POINT_SELECTS $SYSBENCH_SIMPLE_RANGES $SYSBENCH_SUM_RANGES $SYSBENCH_ORDER_RANGES $SYSBENCH_DISTINCT_RANGES $SYSBENCH_INDEX_UPDATES $SYSBENCH_NON_INDEX_UPDATES $SYSBENCH_INSERTS $WRITE_CONCERN $MAX_TPS $MONGO_SERVER $MONGO_PORT $SEED "$USERNAME" "$PASSWORD" $MONGO_READ_PREFERENCE "$TRUST_STORE" "$TRUST_STORE_PASSWORD" $USE_TLS "$MONGODB_REPLICASETNAME" | tee -a $LOG_NAME else echo "executing standard sysbench" | tee -a $LOG_NAME - java -cp $CLASSPATH:$PWD/src jmongosysbenchshardedexecute $NUM_COLLECTIONS $DB_NAME $NUM_WRITER_THREADS $NUM_DOCUMENTS_PER_COLLECTION $NUM_SECONDS_PER_FEEDBACK $BENCHMARK_TSV $SYSBENCH_AUTO_COMMIT $RUN_TIME_SECONDS $SYSBENCH_RANGE_SIZE $SYSBENCH_POINT_SELECTS $SYSBENCH_SIMPLE_RANGES $SYSBENCH_SUM_RANGES $SYSBENCH_ORDER_RANGES $SYSBENCH_DISTINCT_RANGES $SYSBENCH_INDEX_UPDATES $SYSBENCH_NON_INDEX_UPDATES $SYSBENCH_INSERTS $WRITE_CONCERN $MAX_TPS $MONGO_SERVER $MONGO_PORT $SEED "$USERNAME" "$PASSWORD" $MONGO_READ_PREFERENCE "$TRUST_STORE" "$TRUST_STORE_PASSWORD" $USE_TLS $MAX_SHARD_KEY $DOCS_PER_SHARD | tee -a $LOG_NAME + java -cp $CLASSPATH:$PWD/src jmongosysbenchshardedexecute $NUM_COLLECTIONS $DB_NAME $NUM_WRITER_THREADS $NUM_DOCUMENTS_PER_COLLECTION $NUM_SECONDS_PER_FEEDBACK $BENCHMARK_TSV $SYSBENCH_AUTO_COMMIT $RUN_TIME_SECONDS $SYSBENCH_RANGE_SIZE $SYSBENCH_POINT_SELECTS $SYSBENCH_SIMPLE_RANGES $SYSBENCH_SUM_RANGES $SYSBENCH_ORDER_RANGES $SYSBENCH_DISTINCT_RANGES $SYSBENCH_INDEX_UPDATES $SYSBENCH_NON_INDEX_UPDATES $SYSBENCH_INSERTS $WRITE_CONCERN $MAX_TPS $MONGO_SERVER $MONGO_PORT $SEED "$USERNAME" "$PASSWORD" $MONGO_READ_PREFERENCE "$TRUST_STORE" "$TRUST_STORE_PASSWORD" $USE_TLS $MAX_SHARD_KEY $DOCS_PER_SHARD "$MONGODB_REPLICASETNAME" | tee -a $LOG_NAME fi echo "" | tee -a $LOG_NAME diff --git a/src/jmongosysbenchexecute.java b/src/jmongosysbenchexecute.java index 16762b1..7cf402e 100644 --- a/src/jmongosysbenchexecute.java +++ b/src/jmongosysbenchexecute.java @@ -56,6 +56,7 @@ public class jmongosysbenchexecute { public static String trustStore; public static String trustStorePassword; public static String useSSL; + public static String replicaSetName; public static int oltpRangeSize; public static int oltpPointSelects; @@ -77,12 +78,12 @@ public jmongosysbenchexecute() { } public static void main (String[] args) throws Exception { - if (args.length != 28) { + if (args.length != 29) { logMe("*** ERROR : CONFIGURATION ISSUE ***"); logMe("jsysbenchexecute [number of collections] [database name] [number of writer threads] [documents per collection] [seconds feedback] "+ "[log file name] [auto commit Y/N] [runtime (seconds)] [range size] [point selects] "+ "[simple ranges] [sum ranges] [order ranges] [distinct ranges] [index updates] [non index updates] [inserts] [writeconcern] "+ - "[max tps] [server] [port] [seed] [username] [password] [read preference] [trust store] [trust store password] [use ssl]"); + "[max tps] [server] [port] [seed] [username] [password] [read preference] [trust store] [trust store password] [use ssl] [replica set name]"); System.exit(1); } @@ -114,6 +115,7 @@ public static void main (String[] args) throws Exception { trustStore = args[25]; trustStorePassword = args[26]; useSSL = args[27].toLowerCase(); + replicaSetName = args[28]; maxThreadTPS = (maxTPS / writerThreads) + 1; @@ -166,6 +168,7 @@ else if ((myWriteConcern.toLowerCase().equals("w3"))) { logMe(" userName = %s",userName); logMe(" read preference = %s",readPreference); logMe(" use SSL = %s",useSSL); + logMe(" replicaSetName = %s",replicaSetName); /* MongoClientOptions clientOptions = new MongoClientOptions.Builder().connectionsPerHost(2048).socketTimeout(60000).writeConcern(myWC).build(); @@ -182,8 +185,8 @@ else if ((myWriteConcern.toLowerCase().equals("w3"))) { */ //String template = "mongodb://%s:%s@%s:%s/admin?ssl=%s&replicaSet=rs0&readpreference=%s&maxPoolSize=4096"; - String template = "mongodb://%s:%s@%s:%s/admin?ssl=%s&readpreference=%s&maxPoolSize=4096&serverSelectionTimeoutMS=60000&replicaSet=rs0"; - String connectionString = String.format(template, userName, passWord, serverName, serverPort, useSSL, readPreference); + String template = "mongodb://%s:%s@%s:%s/admin?ssl=%s&readpreference=%s&maxPoolSize=4096&serverSelectionTimeoutMS=60000&replicaSet=%s"; + String connectionString = String.format(template, userName, passWord, serverName, serverPort, useSSL, readPreference, replicaSetName); //logMe(" connection string = %s",connectionString); //if (useSSL.equals("true")) { diff --git a/src/jmongosysbenchload.java b/src/jmongosysbenchload.java index 9101f7e..0eff152 100644 --- a/src/jmongosysbenchload.java +++ b/src/jmongosysbenchload.java @@ -46,15 +46,17 @@ public class jmongosysbenchload { public static String trustStorePassword; public static String useSSL; + public static String replicaSetName; + public static int allDone = 0; public jmongosysbenchload() { } public static void main (String[] args) throws Exception { - if (args.length != 18) { + if (args.length != 19) { logMe("*** ERROR : CONFIGURATION ISSUE ***"); - logMe("jsysbenchload [number of collections] [database name] [number of writer threads] [documents per collection] [documents per insert] [inserts feedback] [seconds feedback] [log file name] [compression type] [basement node size (bytes)] [writeconcern] [server] [port] [username] [password] [trust store file] [trust store password] [use ssl]"); + logMe("jsysbenchload [number of collections] [database name] [number of writer threads] [documents per collection] [documents per insert] [inserts feedback] [seconds feedback] [log file name] [compression type] [basement node size (bytes)] [writeconcern] [server] [port] [username] [password] [trust store file] [trust store password] [use ssl] [replica set name]"); System.exit(1); } @@ -76,6 +78,7 @@ public static void main (String[] args) throws Exception { trustStore = args[15]; trustStorePassword = args[16]; useSSL = args[17].toLowerCase(); + replicaSetName = args[18]; WriteConcern myWC = new WriteConcern(); if (myWriteConcern.toLowerCase().equals("acknowledged")) { @@ -113,6 +116,7 @@ else if ((myWriteConcern.toLowerCase().equals("w3"))) { logMe(" Server:Port = %s:%d",serverName,serverPort); logMe(" Username = %s",userName); logMe(" Use SSL = %s",useSSL); + logMe(" Replica set name = %s", replicaSetName); /* MongoClientOptions clientOptions = new MongoClientOptions.Builder().connectionsPerHost(2048).socketTimeout(60000).writeConcern(myWC).build(); @@ -129,11 +133,11 @@ else if ((myWriteConcern.toLowerCase().equals("w3"))) { */ //String template = "mongodb://%s:%s@%s:%s/admin?ssl=%s&replicaSet=rs0&readpreference=%s"; - //String template = "mongodb://%s:%s@%s:%s/admin?ssl=%s&readpreference=%s&maxPoolSize=2000"; - String template = "mongodb://%s:%s@%s:%s/admin?ssl=%s&readpreference=%s&maxPoolSize=2000&replicaSet=rs0&w=3&journal=true"; + //String template = "mongodb://%s:%s@%s:%s/admin?ssl=%s&readpreference=%s&maxPoolSize=2000"; + String template = "mongodb://%s:%s@%s:%s/admin?ssl=%s&readpreference=%s&maxPoolSize=2000&replicaSet=%s&w=3&journal=true"; //String readPreference = "secondaryPreferred"; String readPreference = "primary"; - String connectionString = String.format(template, userName, passWord, serverName, serverPort, useSSL, readPreference); + String connectionString = String.format(template, userName, passWord, serverName, serverPort, useSSL, readPreference, replicaSetName); logMe(" connection string = %s",connectionString); //if (useSSL.equals("true")) { diff --git a/src/jmongosysbenchshardedexecute.java b/src/jmongosysbenchshardedexecute.java index 92bc8e4..b23a5ee 100644 --- a/src/jmongosysbenchshardedexecute.java +++ b/src/jmongosysbenchshardedexecute.java @@ -59,6 +59,7 @@ public class jmongosysbenchshardedexecute { public static String useSSL; public static int maxShardKey; public static int docsPerShard; + public static String replicaSetName; public static int oltpRangeSize; public static int oltpPointSelects; @@ -80,12 +81,13 @@ public jmongosysbenchshardedexecute() { } public static void main (String[] args) throws Exception { - if (args.length != 30) { + if (args.length != 31) { logMe("*** ERROR : CONFIGURATION ISSUE ***"); logMe("jsysbenchshardedexecute [number of collections] [database name] [number of writer threads] [documents per collection] [seconds feedback] "+ "[log file name] [auto commit Y/N] [runtime (seconds)] [range size] [point selects] "+ "[simple ranges] [sum ranges] [order ranges] [distinct ranges] [index updates] [non index updates] [inserts] [writeconcern] "+ - "[max tps] [server] [port] [seed] [username] [password] [read preference] [trust store] [trust store password] [use ssl] [max shard key] [num docs per shard]"); + "[max tps] [server] [port] [seed] [username] [password] [read preference] [trust store] [trust store password] [use ssl] [max shard key] [num docs per shard] " + + "[replica set name]"); System.exit(1); } @@ -119,6 +121,7 @@ public static void main (String[] args) throws Exception { useSSL = args[27].toLowerCase(); maxShardKey = Integer.valueOf(args[28]); docsPerShard = Integer.valueOf(args[29]); + replicaSetName = args[30]; // override numMaxInserts = maxShardKey * docsPerShard; @@ -176,6 +179,7 @@ else if ((myWriteConcern.toLowerCase().equals("w3"))) { logMe(" use SSL = %s",useSSL); logMe(" max shard key = %s",maxShardKey); logMe(" num docs per shard = %s",docsPerShard); + logMe(" replica set nam = %s", replicaSetName); /* MongoClientOptions clientOptions = new MongoClientOptions.Builder().connectionsPerHost(2048).socketTimeout(60000).writeConcern(myWC).build(); diff --git a/src/jmongosysbenchshardedload.java b/src/jmongosysbenchshardedload.java index d043bb8..310abfa 100644 --- a/src/jmongosysbenchshardedload.java +++ b/src/jmongosysbenchshardedload.java @@ -47,6 +47,7 @@ public class jmongosysbenchshardedload { public static String useSSL; public static int maxShardKey; public static int docsPerShard; + public static String replicaSetName; public static int allDone = 0; @@ -54,9 +55,9 @@ public jmongosysbenchshardedload() { } public static void main (String[] args) throws Exception { - if (args.length != 20) { + if (args.length != 21) { logMe("*** ERROR : CONFIGURATION ISSUE ***"); - logMe("jsysbenchload [number of collections] [database name] [number of writer threads] [documents per collection] [documents per insert] [inserts feedback] [seconds feedback] [log file name] [compression type] [basement node size (bytes)] [writeconcern] [server] [port] [username] [password] [trust store file] [trust store password] [use ssl] [max shard key] [docs per shard]"); + logMe("jsysbenchload [number of collections] [database name] [number of writer threads] [documents per collection] [documents per insert] [inserts feedback] [seconds feedback] [log file name] [compression type] [basement node size (bytes)] [writeconcern] [server] [port] [username] [password] [trust store file] [trust store password] [use ssl] [max shard key] [docs per shard] [replica set name]"); System.exit(1); } @@ -80,9 +81,10 @@ public static void main (String[] args) throws Exception { useSSL = args[17].toLowerCase(); maxShardKey = Integer.valueOf(args[18]); docsPerShard = Integer.valueOf(args[19]); + replicaSetName = args[20]; - // override numMaxInserts - numMaxInserts = maxShardKey * docsPerShard; + // override numMaxInserts + numMaxInserts = maxShardKey * docsPerShard; WriteConcern myWC = new WriteConcern(); if (myWriteConcern.toLowerCase().equals("acknowledged")) { @@ -122,6 +124,7 @@ else if ((myWriteConcern.toLowerCase().equals("w3"))) { logMe(" Use SSL = %s",useSSL); logMe(" Maximum Shard Key = %s",maxShardKey); logMe(" Documents Per Shard = %s",docsPerShard); + logMe(" Replica set name = %s", replicaSetName); /* MongoClientOptions clientOptions = new MongoClientOptions.Builder().connectionsPerHost(2048).socketTimeout(60000).writeConcern(myWC).build();