interfisch
/
bobbycar
1
0
Fork 0
Code Issues Pull Requests Releases Wiki Activity

update logfix for new logformat and only output csv lines

This commit is contained in:
interfisch 2023-07-12 18:39:46 +02:00
parent 8dd552111c
commit 33a5669545
1 changed files with 26 additions and 15 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

41
logdata_visualization/logfix.py
View File

@ -23,12 +23,11 @@ with open(inputFilename, 'r') as reader:
lines = [x.rstrip("\n") for x in lines] #remove \n lines = [x.rstrip("\n") for x in lines] #remove \n
commentlines = [True if x.startswith('#') else False for x in lines] #generate mask for lines with comments commentlinesMask = [True if x.startswith('#') else False for x in lines] #generate mask for lines with comments
commentlines[0]= True #first line is a comment everytime
lines=np.array(lines) lines=np.array(lines)
commentlines=np.array(commentlines) commentlinesMask=np.array(commentlinesMask)
datalines = lines[commentlines==False] #get lines with data datalines = lines[commentlinesMask==False] #get lines with data
header = datalines[0] #header is the first non comment line header = datalines[0] #header is the first non comment line
headerSize = len(header.split(',')) #how many elements are expected per line headerSize = len(header.split(',')) #how many elements are expected per line
@ -42,16 +41,25 @@ datalinesFail = datalines[np.array(datalinesSize)!=headerSize]
linesSize = [len(x.split(',')) for x in lines] #count arraysize for every dataline linesSize = [len(x.split(',')) for x in lines] #count arraysize for every dataline
linesOK = np.array(linesSize)==headerSize #mask for okay lines (valid for data lines) linesOK = np.array(linesSize)==headerSize #mask for okay lines (valid for data lines)
timestamp=int(lines[0].split('TIMESTAMP:')[1]) #timestamp when file was created
print("Found "+str(len(lines))+" lines") print("Found "+str(len(lines))+" lines")
print(str(np.sum(commentlines))+" comments") print(str(np.sum(commentlinesMask))+" comments")
print(str(len(datalinesFail))+" Datalines Failed") print(str(len(datalinesFail))+" Datalines Failed")
print(str(len(datalinesOK))+" Datalines OK") print(str(len(datalinesOK))+" Datalines OK")
print("Header Size is "+str(headerSize)) print("Header Size is "+str(headerSize))
timestampline=-1
timestampfound=False
while not timestampfound:
timestampline+=1
timestampfound = (lines[timestampline].find('TIMESTAMP:')!=-1)
timestamp=int(lines[timestampline].split('TIMESTAMP:')[1]) #timestamp when file was created
print("Found Timestamp in line "+str(timestampline))
filetime = time.strftime('%Y%m%d_%H%M%S', time.localtime(timestamp)) filetime = time.strftime('%Y%m%d_%H%M%S', time.localtime(timestamp))
if outputFilename is None: if outputFilename is None:
outputFilename = filetime+".csv" outputFilename = filetime+".csv"
@ -64,15 +72,18 @@ print("Local Time:"+time.strftime('%A, %Y-%m-%d %H:%M:%S', time.localtime(timest
print("Writing to: "+str(outputFilename)) print("Writing to: "+str(outputFilename))
print("Size lines="+str(len(lines)))
print("Size commentlinesMask="+str(len(commentlinesMask)))
print("Size datalines="+str(len(datalines)))
print("Size linesOK="+str(len(linesOK)))
linesWritten = 0 linesWritten = 0
if ok: if ok:
with open(outputFilename, 'w') as writer: with open(outputFilename, 'w') as writer:
for i,line in enumerate(lines): for i,line in enumerate(datalinesOK):
if i!=0 and (commentlines[i] or linesOK[i]): writer.write(line+"\n")
writer.write(line+"\n") linesWritten+=1
linesWritten+=1
else:
print("Skipped "+str(i)+": "+str(line))
print(str(linesWritten)+" lines written to "+str(outputFilename)) print(str(linesWritten)+" lines written to "+str(outputFilename))
else: else: