import processing.video.*; //Video | GStreamer-based video library for Processing //on linux: sudo apt-get install libgstreamer-plugins-base1.0-dev libgstreamer1.0-dev libglib2.0-dev // sudo apt-get install libgstreamer-plugins-good1.0-dev import processing.serial.*; Serial sPort; Capture cam; PImage cam_reference; PImage cam_currentbox; PGraphics canvas; PImage image_clear; int currentID=0; //int MAXID = 50; //inclusive //int MAXID = 599; int MAXID = 10; int searchstate=0; //0=get reference image and ping box, 1=wait, 2=get image to compare and turn box off, 3=wait long starttime_wait=0; float PIXELDISTANCE_CM=1.6; //pixeldistance in centimeter boolean running=false; color testcolor=color(255,255,255); color offcolor=color(0,0,0); color successcolor=color(0,255,0); color failcolor=color(255,0,0); static int boxnum=19; //how many boxes are visible? ArrayList boxes=new ArrayList(); //result will be written to the following arrays //int boxid[] = new int[boxnum]; //for found boxes //PVector boxpos[] = new PVector[boxnum]; //center position of box //PVector boxsize[] = new PVector[boxnum]; //size of box (width,height) //Settings //int FINDBOXES_MINIMUMBRIGHTNESS; //brightness difference required int FINDBOXES_MAXIMUMPIXELDISTANCE; //distance to add pixel to existing box PVector MINBOXSIZE; //minimum box size to accept box int TIME_FIND = 100; //time in ms to find box in image int TIME_RESULT = 100; //time in ms to show success/fail int TIME_TURNOFF = 100; //time in ms to wait to turn off float MINBRIGHTNESS_PERCENTILE=0.98; //percentile for minbrightness calculation (how many pixels will be considered). between 0(all) and 1(none). 0=all pixels. ->1=only bright pixels int MINIMUMALLOWEDBRIGHTNESS=50; //minimum brightness. will be limited to this lower bound if no box in image (percentile calculation yields low value) void setup() { size(640, 480); frameRate(30); /* for (int i=0; i"+cameras[i]); } // The camera can be initialized directly using an // element from the array returned by list(): //cam = new Capture(this, cameras[1]); //windows cam = new Capture(this, cameras[0]); //linux cam.start(); } image_clear=cam.copy(); cam_reference=cam.copy(); } void draw() { if (cam.available() == true) { cam.read(); } //image(cam, 0, 0); // The following does the same, and is faster when just drawing the image // without any additional resizing, transformations, or tint. //set(0, 0, cam); //Subtract images canvas.beginDraw(); canvas.background(0); canvas.blendMode(NORMAL); canvas.image(cam, 0, 0); canvas.blendMode(SUBTRACT); canvas.image(cam_reference, 0, 0); canvas.endDraw(); image(canvas,0,0); //show camera image if (running) { checkState(); } textSize(12); fill(0,255,150); //textcolor text("id: " + currentID, 10, 20); text("state: " + searchstate, 10, 20*2); //draw boxes for (Box box : boxes) { PVector boxcenter = box.getBoxCenter(); PVector boxsize = box.getBoxSize(); rectMode(CENTER); noFill(); fill(255,255,0,100); stroke(255,0,0); rect(boxcenter.x,boxcenter.y, boxsize.x,boxsize.y); textSize(12); fill(0,255,150); //textcolor text(box.getBoxID(), boxcenter.x,boxcenter.y); } } void checkState() { switch(searchstate) { case 0: //0=get reference image and ping box println("Set achterbahn to debug mode"); String writeserial="debug_dark\n"; sPort.write(writeserial); delay(500); cam_reference=cam.copy(); println(); println("Searching BoxID "+str(currentID)); colorBox(currentID); starttime_wait=millis(); searchstate++; break; case 1: //1=wait if (millis()>starttime_wait + TIME_FIND) { //wait some time searchstate++; } break; case 2: //2=get image to compare and turn box off cam_currentbox=canvas.copy(); Box box = findSingleBox(cam_currentbox); //find single box in this image if (box!=null && (box.getBoxSize().x= pow(cam_currentbox.width/40,2)) { //if box is big enough box.setBoxID(currentID); box.setConfidence(1.0); boxes.add(box); } } if (box!=null) { println("Found box"); println("Pos: "+box.getBoxCenter().x+", "+box.getBoxCenter().y+" | size: "+box.getBoxSize().x+", "+box.getBoxSize().y); //colorBox(currentID); }else { println("No Box Found!"); //colorBox(currentID); } starttime_wait=millis(); searchstate++; break; case 3: //3=wait, color box fail or success if (millis()>starttime_wait + TIME_RESULT) { //wait some time colorAllOff(); starttime_wait=millis(); searchstate++; } break; case 4: //wait for box to turn off completely if (millis()>starttime_wait + TIME_TURNOFF) { //wait some time starttime_wait=millis(); searchstate++; } break; case 5: //next box searchstate=0; currentID++; if (currentID>MAXID) { running=false; cam_reference=image_clear; //show normal camera image } break; } } void keyPressed() { if (key == 'n' ){ cam_reference=cam.copy(); println("New Reference Set"); } else if (key == 10){ //ENTER running=true; colorAllOff(); //turn box off currentID=0; //reset searchstate=0; boxes.clear(); //reset found boxes println("Search started"); } else if (key == 'r' ){ running=false; cam_reference=image_clear; //show normal camera image } else if(key == 's') { sendHeights(); } else if(key == 'S') { saveToEEPROM(); } else if(key == 'p') { calculateConfidences(); } } int getPercentileBrightness(PImage img, float percentile) { //percentile between 0 and 1 percentile=min(max(percentile,0.0),1.0); //limit to 0 to 1 int[] brightness=new int[img.pixels.length]; for (int i=0;i findBoxes(PImage img) { ArrayList foundboxes=new ArrayList(); int minimumbrightness=getPercentileBrightness(img, MINBRIGHTNESS_PERCENTILE); //calculate brightness by percentile based on minimum box size minimumbrightness=max(minimumbrightness, MINIMUMALLOWEDBRIGHTNESS); //limit lower value println(" minimumbrightness="+minimumbrightness); for (int y=0;yminimumbrightness) { //pixel has changed Box nearestBox=null; float nearestBoxDistance=MAX_FLOAT; for (Box cb : foundboxes) { //check all known boxes float cdist = cb.getDistanceTo(new PVector(x,y)); if (cdist<=FINDBOXES_MAXIMUMPIXELDISTANCE && cdist0.5) { writeserial="setpx="+box.getBoxID()+","+int(pixelvalue)+"\n"; sPort.write(writeserial); } } println("send"); } void calculateConfidences() { for (int i=0;iboxes.get(prevBoxi).getBoxID()) { //new one closer to boxid prevBoxi=j; } } if (boxes.get(j).getBoxID()>boxid) { //found if (nextBoxi<0 || boxes.get(j).getBoxID()=0) { PVector prevboxcenter=boxes.get(prevBoxi).getBoxCenter(); prevdist=dist(prevboxcenter.x,prevboxcenter.y,boxcenter.x,boxcenter.y); int boxsteps = boxid-boxes.get(prevBoxi).getBoxID(); //1 if previous box found. greater if boxes missing in between prevdist*=boxsteps; } float nextdist=0; if (nextBoxi>=0) { PVector nextboxcenter=boxes.get(nextBoxi).getBoxCenter(); nextdist=dist(nextboxcenter.x,nextboxcenter.y,boxcenter.x,boxcenter.y); int boxsteps = boxes.get(nextBoxi).getBoxID()-boxid; //1 if previous box found. greater if boxes missing in between prevdist*=boxsteps; } float maxdist=max(prevdist,nextdist); //maximum distance to neighbor if one exists maxdist/=cam.width; float confidence=constrain(map(maxdist, 0.05, 0.5, 1.0,0.1), 0.1,1.0); confidence*=constrain(map(boxarea, 1000, 10000, 1.0,0.5), 0.5,1.0); box.setConfidence(confidence); println(boxid+":"+"maxdist="+maxdist+", area="+boxarea+", confidence="+confidence); } } float getMeanDist() { //run after confidences have been calculated float meandist=0; int meancount=0; for (int i=0;i0.5) { //confidence high enough int boxid=box.getBoxID(); PVector boxcenter=box.getBoxCenter(); PVector boxsize=box.getBoxSize(); //calculate distance to neighboring boxes int prevBoxi=-1; int nextBoxi=-1; for (int j=0;jboxes.get(prevBoxi).getBoxID()) { //new one closer to boxid prevBoxi=j; } } if (boxes.get(j).getBoxID()>boxid) { //found if (nextBoxi<0 || boxes.get(j).getBoxID()=0) { PVector prevboxcenter=boxes.get(prevBoxi).getBoxCenter(); prevdist=dist(prevboxcenter.x,prevboxcenter.y,boxcenter.x,boxcenter.y); int boxsteps = boxid-boxes.get(prevBoxi).getBoxID(); //1 if previous box found. greater if boxes missing in between prevdist*=boxsteps; } float nextdist=0; if (nextBoxi>=0) { PVector nextboxcenter=boxes.get(nextBoxi).getBoxCenter(); nextdist=dist(nextboxcenter.x,nextboxcenter.y,boxcenter.x,boxcenter.y); int boxsteps = boxes.get(nextBoxi).getBoxID()-boxid; //1 if previous box found. greater if boxes missing in between prevdist*=boxsteps; } mindist=max(prevdist,nextdist); //maximum distance to neighbor if one exists } if (mindist>0){ meancount++; meandist+=mindist; } } //end for all boxes return meandist/meancount; } void saveToEEPROM() { String writeserial="save\n"; sPort.write(writeserial); delay(500); println("send save to eeprom"); }