From 33a5669545457828ecd32fca407cceb3d1d86f2d Mon Sep 17 00:00:00 2001 From: Fisch Date: Wed, 12 Jul 2023 18:39:46 +0200 Subject: [PATCH] update logfix for new logformat and only output csv lines --- logdata_visualization/logfix.py | 41 +++++++++++++++++++++------------ 1 file changed, 26 insertions(+), 15 deletions(-) diff --git a/logdata_visualization/logfix.py b/logdata_visualization/logfix.py index 3fbafae..a603252 100644 --- a/logdata_visualization/logfix.py +++ b/logdata_visualization/logfix.py @@ -23,12 +23,11 @@ with open(inputFilename, 'r') as reader: lines = [x.rstrip("\n") for x in lines] #remove \n -commentlines = [True if x.startswith('#') else False for x in lines] #generate mask for lines with comments -commentlines[0]= True #first line is a comment everytime +commentlinesMask = [True if x.startswith('#') else False for x in lines] #generate mask for lines with comments lines=np.array(lines) -commentlines=np.array(commentlines) -datalines = lines[commentlines==False] #get lines with data +commentlinesMask=np.array(commentlinesMask) +datalines = lines[commentlinesMask==False] #get lines with data header = datalines[0] #header is the first non comment line headerSize = len(header.split(',')) #how many elements are expected per line @@ -42,16 +41,25 @@ datalinesFail = datalines[np.array(datalinesSize)!=headerSize] linesSize = [len(x.split(',')) for x in lines] #count arraysize for every dataline linesOK = np.array(linesSize)==headerSize #mask for okay lines (valid for data lines) -timestamp=int(lines[0].split('TIMESTAMP:')[1]) #timestamp when file was created - - - print("Found "+str(len(lines))+" lines") -print(str(np.sum(commentlines))+" comments") +print(str(np.sum(commentlinesMask))+" comments") print(str(len(datalinesFail))+" Datalines Failed") print(str(len(datalinesOK))+" Datalines OK") print("Header Size is "+str(headerSize)) +timestampline=-1 +timestampfound=False +while not timestampfound: + timestampline+=1 + timestampfound = (lines[timestampline].find('TIMESTAMP:')!=-1) + +timestamp=int(lines[timestampline].split('TIMESTAMP:')[1]) #timestamp when file was created + +print("Found Timestamp in line "+str(timestampline)) + + + + filetime = time.strftime('%Y%m%d_%H%M%S', time.localtime(timestamp)) if outputFilename is None: outputFilename = filetime+".csv" @@ -64,15 +72,18 @@ print("Local Time:"+time.strftime('%A, %Y-%m-%d %H:%M:%S', time.localtime(timest print("Writing to: "+str(outputFilename)) +print("Size lines="+str(len(lines))) +print("Size commentlinesMask="+str(len(commentlinesMask))) +print("Size datalines="+str(len(datalines))) +print("Size linesOK="+str(len(linesOK))) + + linesWritten = 0 if ok: with open(outputFilename, 'w') as writer: - for i,line in enumerate(lines): - if i!=0 and (commentlines[i] or linesOK[i]): - writer.write(line+"\n") - linesWritten+=1 - else: - print("Skipped "+str(i)+": "+str(line)) + for i,line in enumerate(datalinesOK): + writer.write(line+"\n") + linesWritten+=1 print(str(linesWritten)+" lines written to "+str(outputFilename)) else: