new db changes

portal/pace/datastructs.py

@@ -4,7 +4,8 @@
 from sqlalchemy.dialects.mysql import MEDIUMTEXT,INTEGER,DECIMAL
 from datetime import datetime
 
-class Timingprofile(db.Model):
+class E3SMexp(db.Model):
+	__tablename__ = 'e3smexp'
 	expid = db.Column(INTEGER(unsigned=True), primary_key=True, index=True)     
 	case = db.Column(db.String(100),nullable=False, index=True)
 	lid = db.Column(db.String(50), nullable=False)
@@ -31,11 +32,12 @@ class Timingprofile(db.Model):
 	run_time = db.Column(DECIMAL(20,3,unsigned=True), nullable=False)
 	final_time = db.Column(DECIMAL(10,3,unsigned=True), nullable=False)
 	version = db.Column(db.String(100),nullable=False)
-
+	upload_by = db.Column(db.String(25),nullable=False, default='sarat')
 
 class Pelayout(db.Model):
+	__tablename__ = 'pelayout'
 	id = db.Column(INTEGER(unsigned=True), primary_key=True)
-	expid = db.Column(INTEGER(unsigned=True), db.ForeignKey('timingprofile.expid'), nullable=False, index=True)
+	expid = db.Column(INTEGER(unsigned=True), db.ForeignKey('e3smexp.expid'), nullable=False, index=True)
 	component = db.Column(db.String(25), nullable=False, index=True)
 	comp_pes = db.Column(INTEGER(unsigned=True), nullable=False)
 	root_pe = db.Column(INTEGER(unsigned=True), nullable=False)
@@ -45,25 +47,28 @@ class Pelayout(db.Model):
 	stride = db.Column(INTEGER(unsigned=True), nullable=False)
 
 class Runtime(db.Model):
+	__tablename__ = 'runtime'
 	id = db.Column(INTEGER(unsigned=True), primary_key=True)
-	expid = db.Column(INTEGER(unsigned=True), db.ForeignKey('timingprofile.expid'), nullable=False, index=True)
+	expid = db.Column(INTEGER(unsigned=True), db.ForeignKey('e3smexp.expid'), nullable=False, index=True)
 	component = db.Column(db.String(25), nullable=False, index=True)
 	seconds = db.Column(DECIMAL(10,3,unsigned=True), nullable=False)
 	model_day = db.Column(DECIMAL(10,3,unsigned=True), nullable=False)
 	model_years = db.Column(DECIMAL(10,2,unsigned=True), nullable=False)
 
 
 class ModelTiming(db.Model):
+	__tablename__ = 'model_timing'
 	id = db.Column(INTEGER(unsigned=True), primary_key=True,autoincrement=True)
-	expid = db.Column(INTEGER(unsigned=True), db.ForeignKey('timingprofile.expid'), nullable=False, index=True)
+	expid = db.Column(INTEGER(unsigned=True), db.ForeignKey('e3smexp.expid'), nullable=False, index=True)
 	jsonVal = db.Column(MEDIUMTEXT, nullable=False)
 	rank = db.Column(db.String(10), nullable=False)
 
 class Authusers(db.Model):
 	id = db.Column(INTEGER(unsigned=True), primary_key=True,autoincrement=True)
 	user = db.Column(db.String(50))
 
-class Additionalnote(db.Model):
+class Expnotes(db.Model):
+	__tablename__ = 'expnotes'
 	id = db.Column(INTEGER(unsigned=True), primary_key=True, index=True)
-	expid = db.Column(INTEGER(unsigned=True), db.ForeignKey('timingprofile.expid'), nullable=False, index=True)
+	expid = db.Column(INTEGER(unsigned=True), db.ForeignKey('e3smexp.expid'), nullable=False, index=True)
 	note = db.Column(MEDIUMTEXT, nullable=False)

portal/pace/parse.py

@@ -16,10 +16,10 @@
 from minio.error import (ResponseError, BucketAlreadyOwnedByYou,BucketAlreadyExists)
 
 # main
-def parseData(zipfilename,user):
+def parseData(zipfilename,uploaduser):
 	# open file to write pace report
 	old_stdout = sys.stdout
-	logfilename = 'pace-'+str(user)+'-'+str(datetime.now().strftime('%Y-%m-%d-%H:%M:%S'))+'.log'
+	logfilename = 'pace-'+str(uploaduser)+'-'+str(datetime.now().strftime('%Y-%m-%d-%H:%M:%S'))+'.log'
 	logfile = PACE_LOG_DIR + logfilename
 	log_file = open(logfile,'w')
 	sys.stdout = log_file
@@ -89,7 +89,7 @@ def parseData(zipfilename,user):
 		print (' ')
 		print ('**************************************************')
 		# insert experiments for given files
-		isSuccess.append(insertExperiment(allfile[i],readmefile[i],timingfile[i],gitdescribefile[i],db,fpath))
+		isSuccess.append(insertExperiment(allfile[i],readmefile[i],timingfile[i],gitdescribefile[i],db,fpath, uploaduser))
 		print ('**************************************************')
 		print (' ')
 
@@ -167,7 +167,7 @@ def convertPathtofile(path):
 # function to check duplicate experiments (check based on user,machinr,exp_date,case)
 def checkDuplicateExp(euser,emachine,ecurr, ecase):
 	eexp_date = changeDateTime(ecurr)
-	flag=Timingprofile.query.filter_by(user=euser,machine=emachine,case=ecase,exp_date=eexp_date ).first()
+	flag=E3SMexp.query.filter_by(user=euser,machine=emachine,case=ecase,exp_date=eexp_date ).first()
 	if flag is None:
 		return(False)
 	else:
@@ -241,10 +241,10 @@ def parseModelVersion(gitfile):
 	return version
 
 # This function provides pathway to files for their respective parser function and finally stores in database
-def insertExperiment(filename,readmefile,timingfile,gitfile,db,fpath):
+def insertExperiment(filename,readmefile,timingfile,gitfile,db,fpath, uploaduser):
 
 	# returns expid if success else returns False
-	forexpid = parseE3SMtiming(filename,readmefile,gitfile,db,fpath)
+	forexpid = parseE3SMtiming(filename,readmefile,gitfile,db,fpath, uploaduser)
 	if forexpid == False:
 		return False
 	print ('* Parsing: '+ convertPathtofile(timingfile))
@@ -284,7 +284,7 @@ def insertExperiment(filename,readmefile,timingfile,gitfile,db,fpath):
 	return (True) 
 
 # parse e3sm files
-def parseE3SMtiming(filename,readmefile,gitfile,db,fpath):
+def parseE3SMtiming(filename,readmefile,gitfile,db,fpath, uploaduser):
 	# open file
 	if filename.endswith('.gz'):
 		parseFile=gzip.open(filename,'rb')
@@ -473,7 +473,7 @@ def parseE3SMtiming(filename,readmefile,gitfile,db,fpath):
 		print ('    -Complete')
 
 		# insert timingprofile 
-		new_experiment = Timingprofile(case=timingProfileInfo['case'],
+		new_experiment = E3SMexp(case=timingProfileInfo['case'],
 						lid=timingProfileInfo['lid'],
 						machine=timingProfileInfo['machine'],
 						caseroot=timingProfileInfo['caseroot'],
@@ -496,11 +496,12 @@ def parseE3SMtiming(filename,readmefile,gitfile,db,fpath):
 						init_time=timingProfileInfo['init_time'],
 						run_time=timingProfileInfo['run_time'],
 						final_time=timingProfileInfo['final_time'],
-						version = expversion)
+						version = expversion,
+						upload_by = uploaduser)
 		db.session.add(new_experiment)
 
 		# table has to have a same experiment id
-		forexpid = Timingprofile.query.order_by(Timingprofile.expid.desc()).first()
+		forexpid = E3SMexp.query.order_by(E3SMexp.expid.desc()).first()
 
 		#insert pelayout
 		i=0

portal/pace/webapp.py

@@ -245,7 +245,7 @@ def summaryQuery(expID,rank):
     else:
         resultNodes = db.engine.execute("select jsonVal from model_timing where expid = "+str(expID)+ " and rank = '"+rank+"'").fetchall()[0].jsonVal
         #Get user and machine information:
-        tpData = db.engine.execute("select compset,res from timingprofile where expid = "+str(expID)).fetchall()
+        tpData = db.engine.execute("select compset,res from e3smexp where expid = "+str(expID)).fetchall()
         compset,res = tpData[0].compset,tpData[0].res
 
     if rank == 'stats':
@@ -271,7 +271,7 @@ def summaryQuery(expID,rank):
 def expDetails(mexpid):
 	myexp = None
 	try:
-		myexp = db.engine.execute("select * from timingprofile where expid= "+ str(mexpid) ).fetchall()[0]
+		myexp = db.engine.execute("select * from e3smexp where expid= "+ str(mexpid) ).fetchall()[0]
 	except IndexError:
 		return render_template('error.html')
 	mypelayout = db.engine.execute("select * from pelayout where expid= "+ str(mexpid) ).fetchall()
@@ -281,7 +281,7 @@ def expDetails(mexpid):
 	for i in range(len(runtimeSvg.default_args['comps'])):
 		colorDict[runtimeSvg.default_args['comps'][i]] = runtimeSvg.default_args['color'][i]
 	try:
-		noteexp = db.engine.execute("select * from additionalnote where expid= "+ str(mexpid) ).fetchall()[0]
+		noteexp = db.engine.execute("select * from expnotes where expid= "+ str(mexpid) ).fetchall()[0]
 		note = noteexp.note
 	except IndexError:
 		note=""
@@ -297,22 +297,22 @@ def note(expID):
 	if userlogin == True:
 		if request.method == "GET":
 			try:
-				myexpid = db.engine.execute("select * from timingprofile where expid= "+expID).fetchall()[0]
+				myexpid = db.engine.execute("select * from e3smexp where expid= "+expID).fetchall()[0]
 			except IndexError:
 				return render_template('error.html')
 			try:
-				myexp = db.engine.execute("select * from additionalnote where expid= "+expID).fetchall()[0]
+				myexp = db.engine.execute("select * from expnotes where expid= "+expID).fetchall()[0]
 				note = myexp.note
 			except IndexError:
 				note=""
 			return render_template('note.html', note = note, expid = expID)
 		elif request.method == "POST":
 			note = request.form['note']		
 			try:
-				myexp = db.engine.execute("select * from additionalnote where expid= "+expID).fetchall()[0]
-				db.engine.execute("update additionalnote set note =\'"+str(note)+"\' where expid = " +expID)
+				myexp = db.engine.execute("select * from expnotes where expid= "+expID).fetchall()[0]
+				db.engine.execute("update expnotes set note =\'"+str(note)+"\' where expid = " +expID)
 			except IndexError:
-				db.engine.execute("insert into additionalnote(expid,note) values ("+expID+",\'"+str(note)+"\')")
+				db.engine.execute("insert into expnotes(expid,note) values ("+expID+",\'"+str(note)+"\')")
 			return redirect('/exp-details/'+str(expID))
 	else:
 		return redirect('/login')
@@ -350,7 +350,7 @@ def searchPage(searchQuery="*",isHomePage=False):
 def advSearch(searchQuery):
     return searchPage(searchQuery,False)
 
-#This is a rest-like API that gives information about queried experiments from the timingprofile table.
+#This is a rest-like API that gives information about queried experiments from the e3smexp table.
 @app.route("/ajax/search/<searchTerms>")
 @app.route("/ajax/search/<searchTerms>/<limit>")
 @app.route("/ajax/search/<searchTerms>/<limit>/")
@@ -363,7 +363,7 @@ def searchCore(searchTerms,limit = False,orderBy="expid",ascDsc="desc",whiteList
     #Variable names are split into non-string and string respectively; this is to help improve search results during a basic search.
     variableList=[
         ["expid","total_pes_active","run_length","model_throughput","mpi_tasks_per_node","init_time","run_time"],
-        ["user","machine","compset","exp_date","res","timingprofile.case"]
+        ["user","machine","compset","exp_date","res","e3smexp.case"]
     ]
 
     specificVariables = whiteList
@@ -372,7 +372,7 @@ def searchCore(searchTerms,limit = False,orderBy="expid",ascDsc="desc",whiteList
 
     #This should be an easy way to determine if something's in the list
     if orderBy == "case":
-        orderBy = "timingprofile.case"
+        orderBy = "e3smexp.case"
     elif orderBy not in variableList[0] + variableList[1]:
         orderBy = "expid"
     #Only asc and desc are allowed:
@@ -417,7 +417,7 @@ def searchCore(searchTerms,limit = False,orderBy="expid",ascDsc="desc",whiteList
 
     #The original version of searchCore did not have these three functions separated... this is for cleaner code XP
     def searchAll(termString):
-        queryStr = "select "+str(specificVariables).strip("[]").replace("'","")+" from timingprofile order by "+orderBy+" "+ascDsc
+        queryStr = "select "+str(specificVariables).strip("[]").replace("'","")+" from e3smexp order by "+orderBy+" "+ascDsc
         if limit:
             queryStr+=" limit "+limit
         allResults = db.engine.execute(queryStr).fetchall()
@@ -443,8 +443,8 @@ def advSearch(termString):
             if syntax[0] in variableList[0] + variableList[1]:
                 strList.append(syntax[0]+elementStr)
             elif syntax[0] == "case":
-                strList.append('timingprofile.case '+elementStr)
-        compiledString = "select " + str(specificVariables).strip("[]").replace("'","") + " from timingprofile where "
+                strList.append('e3smexp.case '+elementStr)
+        compiledString = "select " + str(specificVariables).strip("[]").replace("'","") + " from e3smexp where "
         for i in range(len(strList)):
             compiledString+=strList[i]
             if not i==len(strList) - 1:
@@ -469,7 +469,7 @@ def basicSearch(termString):
         #print(termList)
 
         if(len(termList) > 0):
-            queryStr = "select " + str(specificVariables).strip("[]").replace("'","") + " from timingprofile where "
+            queryStr = "select " + str(specificVariables).strip("[]").replace("'","") + " from e3smexp where "
             for term in termList:
                 #This controlls whether or not to search through number-based or string based variables:
                 targetIndex = 0
@@ -546,12 +546,12 @@ def specificSearch(query,whiteList = "total_pes_active,model_throughput,machine,
 def searchSummary(query = ""):
     return render_template("searchSummary.html",query=query)
 
-#Get a specific list of elements from timingprofile. Only specific elements are allowed, so users cannot grab everything.
+#Get a specific list of elements from e3smexp. Only specific elements are allowed, so users cannot grab everything.
 @app.route("/ajax/getDistinct/<entry>")
 def getDistinct(entry):
     queryList = []
     if entry in ["machine","user"]:
-        distQuery = db.engine.execute("select distinct "+entry+" from timingprofile order by "+entry).fetchall()
+        distQuery = db.engine.execute("select distinct "+entry+" from e3smexp order by "+entry).fetchall()
         for element in distQuery:
             queryList.append(element[entry])
     return json.dumps(queryList)
@@ -577,13 +577,13 @@ def searchPrediction(keyword):
     #The keyword is designed to be a single word without any potential database loopholes:
     keyword = keyword.replace("\\c","").replace(";","").replace(" ","")
     #Grab elements based on these columns:
-    columnNames = ["user","machine","expid","compset","res","timingprofile.case"]
+    columnNames = ["user","machine","expid","compset","res","e3smexp.case"]
     resultWords = []
     for column in columnNames:
         colName = column
         if "." in column:
             colName = colName.split(".")[1]
-        distQuery = db.engine.execute("select distinct "+column+" from timingprofile where "+column+" like '"+keyword+"%%' limit 10").fetchall()
+        distQuery = db.engine.execute("select distinct "+column+" from e3smexp where "+column+" like '"+keyword+"%%' limit 10").fetchall()
         for element in distQuery:
             resultWords.append(str(element[colName]))
     #Sort them by similar name: