add rtc to teensy and add python logfile renamer and fixer

controller_teensy/src/main.cpp

@@ -1,6 +1,7 @@
 #include <Arduino.h>
 
 
+
 /*
 Connections:
 Tennsy Pin, Pin Name, Connected to
@@ -192,6 +193,9 @@ void readButtons();
 
 uint16_t linearizeThrottle(uint16_t v);
 
+#include <TimeLib.h> //for teensy rtc 
+time_t getTeensy3Time();
+
 void SendSerial(SerialCommand &scom, int16_t uSpeedLeft, int16_t uSpeedRight, HardwareSerial &SerialRef)
 {
   // Create command
@@ -294,8 +298,16 @@ void setup()
   pinMode(PIN_LATCH_ENABLE, OUTPUT);
   digitalWrite(PIN_LATCH_ENABLE,HIGH); //latch on
   pinMode(PIN_MODE_SWITCH, INPUT_PULLUP);
-}
 
+  setSyncProvider(getTeensy3Time); //See https://www.pjrc.com/teensy/td_libs_Time.html#teensy3
+  if (timeStatus()!= timeSet) {
+    Serial.println("Unable to sync with the RTC");
+  } else {
+    Serial.println("RTC has set the system time");
+  }
+
+  
+}
 
 unsigned long loopmillis;
 // ########################## LOOP ##########################
@@ -347,6 +359,11 @@ void loop() {
 
 // ##### HELPFUNCTIONS
 
+time_t getTeensy3Time()
+{
+  return Teensy3Clock.get();
+}
+
 
 int sort_desc(const void *cmp1, const void *cmp2) //compare function for qsort
 {
@@ -373,6 +390,10 @@ float filterMedian(int16_t* values) {
 }
 
 
+void writeLogInfo(HardwareSerial &SerialRef) { //first line of file
+  SerialRef.print("#TIMESTAMP:");
+  SerialRef.println(now()); 
+}
 
 void writeLogHeader(HardwareSerial &SerialRef) {
   SerialRef.print("time,cmd_FrontL,cmd_FrontR,cmd_RearL,cmd_RearR,");
@@ -602,6 +623,7 @@ void sendCMD() {
 
 void checkLog() {
   if (!log_header_written && loopmillis>=WRITE_HEADER_TIME){ //write header for log file after logger booted up
+    writeLogInfo(Serial1);
     writeLogHeader(Serial1);
     log_header_written=true;
   }

logdata_visualization/logdata_visualization.pde

@@ -1,14 +1,8 @@
 import processing.serial.*;
 
-/*
-TODO
-- in teensy controller: erst table header screiben, dann anfangen zu loggen (oder erste zeilen comments hier ignorieren)
-     - comments werden weiterhin vor header geschrieben.
-*/
-
 int vis_textsize=12; //copy from Visualization class
 
-String logfile_name="/media/fisch/HDD/Projects/bobbycar/bobbycar_repo/logdata_visualization/202107_Thereisacamp/LOG00244.TXT";
+String logfile_name="/media/fisch/HDD/Projects/bobbycar/bobbycar_repo/logdata_visualization/logfixer/20211004_171509.csv";
 int columnCount=20;
 
 boolean useSerial=false; //false=read from csv log, true=read from serial port

logdata_visualization/logfix.py

@@ -0,0 +1,67 @@
+import numpy as np
+from datetime import datetime
+import time
+
+ok=True
+
+inputFilename='LOG00251.TXT'
+
+
+print("Filename: "+str(inputFilename))
+
+with open(inputFilename, 'r') as reader:
+    lines = reader.readlines()
+
+
+lines = [x.rstrip("\n") for x in lines] #remove \n
+commentlines = [True if x.startswith('#') else False for x in lines] #generate mask for lines with comments
+commentlines[0]= True #first line is a comment everytime
+
+lines=np.array(lines)
+commentlines=np.array(commentlines)
+datalines = lines[commentlines==False] #get lines with data
+header = datalines[0] #header is the first non comment line
+
+headerSize = len(header.split(',')) #how many elements are expected per line
+
+datalinesSize = [len(x.split(',')) for x in datalines] #count arraysize for every dataline
+datalinesOK = datalines[np.array(datalinesSize)==headerSize]
+datalinesFail = datalines[np.array(datalinesSize)!=headerSize]
+#datalinesSizeBin = dict((x,datalinesSize.count(x)) for x in set(datalinesSize)) #binning
+#normalSize = max(datalinesSizeBin, key=datalinesSizeBin.get) #get normal element count by highest bin
+
+linesSize = [len(x.split(',')) for x in lines] #count arraysize for every dataline
+linesOK = np.array(linesSize)==headerSize #mask for okay lines (valid for data lines)
+
+timestamp=int(lines[0].split('TIMESTAMP:')[1]) #timestamp when file was created
+#outputFilename=
+
+
+print("Found "+str(len(lines))+" lines")
+print(str(np.sum(commentlines))+" comments")
+print(str(len(datalinesFail))+" Datalines Failed")
+print(str(len(datalinesOK))+" Datalines OK")
+print("Header Size is "+str(headerSize))
+
+filetime = time.strftime('%Y%m%d_%H%M%S', time.localtime(timestamp))
+outputFilename = filetime+".csv"
+
+#is_dst(datetime(2019, 4, 1), timezone="US/Pacific")
+print("Timestamp:"+str(timestamp)+" -> "+str(filetime))
+print("UTC: "+ datetime.utcfromtimestamp(timestamp).strftime('%A, %Y-%m-%d %H:%M:%S'))
+print("Local Time:"+time.strftime('%A, %Y-%m-%d %H:%M:%S', time.localtime(timestamp)))
+
+
+print("Writing to: "+str(outputFilename))
+
+linesWritten = 0
+with open(outputFilename, 'w') as writer:
+    for i,line in enumerate(lines):
+        if i!=0 and (commentlines[i] or linesOK[i]):
+            writer.write(line+"\n")
+            linesWritten+=1
+        else:
+            print("Skipped "+str(i)+": "+str(line))
+        
+print(str(linesWritten)+" lines written to "+str(outputFilename))
+