Face OSC Lip Lyric Creator

For the midterm, I created a tool with Processing and Face OSC that allows you to generate lyrics with your mouth. In order to generate lyrics, you just to open your mouth widely, and then the lyrics will be displayed, line by line, each time your mouth is opened. The lyrics will be displayed in a separate window, and move along with wherever your head and lips move. In order to add lyrics to the sketch, it is very simple, and can be done just by typing in each line in quotes and putting the entire song lyrics inside brackets. The colors of the backgrounds can also be changed to fit the mood of a song. I was motivated to create this project because I love to lip synch along to songs, and I thought creating this tool would take the process of mouthing along to songs to a whole new level.

I learned it was a little difficult with a slower computer that may not register my mouth opening quickly because it would run through the lyrics when I just wanted one line of a song. But, once you get the hang of it, I think this is a really fun tool for “singing” along to your favorite songs, even if you look like you’re a fish gasping for air at the same time. It can also be used for a performance (if someone really knew how to time it well), or a game to see if you can open your mouth at the right time.

Here’s my inspiration/a related project:

As well as the code for this sketch, which I could not figure out how to use, but was a good example to see what she did (I tried to get Syphon, but that was more complicated than necessary):

import codeanticode.syphon.*;
	import oscP5.*;
	OscP5 oscP5;
	SyphonClient client;
	int found;
	float[] rawPoints = new float[131];
	PImage img;
	PImage[] create = new PImage[3];
	float cWidth;
	float cHeight;
	boolean unique;
	PImage currentImg;
	int currentImgIndex;
	public void setup() {
	unique = true;
	create[0] = loadImage(“img1.png“);
	create[1] = loadImage(“img2.png“);
	create[2] = loadImage(“img3.png“);
	currentImgIndex=0;
	currentImg = create[0];
	cWidth = create[0].width;
	cHeight = create[0].height;
	size(640, 480, P3D);
	frameRate(30);
	// Create syhpon client to receive frames
	// from running server with given name:
	oscP5 = new OscP5(this, 8338);
	oscP5.plug(this, “found“, “/found“);
	oscP5.plug(this, “rawRecieved“, “/raw“);
	client = new SyphonClient(this, “FaceOSC“);
	background(0);
	}
	public void draw() {
	if (client.available()) {
	// The first time getImage() is called with
	// a null argument, it will initialize the PImage
	// object with the correct size.
	img = client.getImage(img); // load the pixels array with the updated image info (slow)
	//img = client.getImage(img, false); // does not load the pixels array (faster)
	background(img);
	//background(255,0,0,0);
	if(found > 0) {
	float lastNum = 0;
	int Idx = 0;
	/*
	for (int i=100; i< 131; i++){
	float currentNum = rawPoints[i];
	if( i == 0 || i%2==0 ){
	lastNum = currentNum;
	Idx = i;
	}
	else {
	ellipse(lastNum, currentNum, 2, 2);
	String id = str ( Idx);
	String cur = str ( i);
	String cords = “x: ” + id + “, y: ” + cur +””;
	text (cords, lastNum, currentNum);
	}
	}
	*/
	//ellipse (rawPoints[92],rawPoints[93], 2,2);
	// ellipse (rawPoints[94],rawPoints[95], 2,2);
	/*
	fill(255,0,0);
	ellipse (rawPoints[96],rawPoints[97], 2,2);
	//ellipse (rawPoints[98],rawPoints[99], 2,2);
	//ellipse (rawPoints[100],rawPoints[101], 2,2);
	//ellipse (rawPoints[102],rawPoints[103], 2,2);
	// fill(0,0,255);
	// ellipse (rawPoints[104],rawPoints[105], 2,2);
	//fill(0,0,255);
	//ellipse (rawPoints[106],rawPoints[107], 2,2);
	// fill(0,255,0);
	ellipse (rawPoints[108],rawPoints[109], 2,2);
	//ellipse (rawPoints[110],rawPoints[111], 2,2);
	//ellipse (rawPoints[112],rawPoints[113], 2,2);
	//ellipse (rawPoints[114],rawPoints[115], 2,2);
	//ellipse (rawPoints[116],rawPoints[117], 2,2);
	//ellipse (rawPoints[118],rawPoints[119], 2,2);
	// fill(0,0,255);
	ellipse (rawPoints[120],rawPoints[121], 2,2);
	ellipse (rawPoints[122],rawPoints[123], 2,2);
	ellipse (rawPoints[124],rawPoints[125], 2,2);
	ellipse (rawPoints[126],rawPoints[127], 2,2);
	ellipse (rawPoints[128],rawPoints[129], 2,2);
	*/
	/*
	String cords = “1”;
	text (cords, rawPoints[96],rawPoints[97]);
	cords = “2”;
	text (cords, rawPoints[122],rawPoints[123]);
	cords = “3”;
	text (cords, rawPoints[124],rawPoints[125]);
	cords = “4”;
	text (cords, rawPoints[120],rawPoints[121]);
	cords = “5”;
	text (cords, rawPoints[128],rawPoints[129]);
	cords = “6”;
	text (cords, rawPoints[126],rawPoints[127]);
	cords = “7”;
	text (cords, rawPoints[108],rawPoints[109]);
	println (“the image is width ” + cWidth);
	*/
	beginShape();
	noStroke();
	texture(currentImg);
	//1
	vertex(rawPoints[96],rawPoints[97], 0, cHeight/2 );
	//4
	vertex(rawPoints[120],rawPoints[121], cWidth/4 , 0);
	//2
	vertex(rawPoints[122],rawPoints[123], cWidth/2, 0);
	//3
	vertex(rawPoints[124],rawPoints[125], cWidth/4 * 3, 0);
	//7
	vertex(rawPoints[108],rawPoints[109], cWidth , cHeight/2);
	//6
	vertex(rawPoints[126],rawPoints[127],cWidth/4 * 3, cHeight);
	//5
	vertex(rawPoints[128],rawPoints[129],cWidth/2, cHeight);
	//1
	//vertex(rawPoints[96],rawPoints[97],0, cHeight/2 );
	endShape(CLOSE);
	if ((dist(rawPoints[122],rawPoints[123],rawPoints[128],rawPoints[129]) < 5) && (unique == true) ){
	println (“!!!!!“);
	if ( currentImgIndex < create.length–1){
	currentImgIndex ++;
	}
	else {
	currentImgIndex = 0;
	}
	currentImg = create[currentImgIndex];
	cWidth = currentImg.width;
	cHeight = currentImg.height;
	unique = false;
	}
	else if (dist(rawPoints[122],rawPoints[123],rawPoints[128],rawPoints[129]) > 5 ) {
	unique = true;
	}
	// ellipse (rawPoints[130],rawPoints[131], 2,2);
	}
	else{
	println (“not found“);
	}
	}
	}
	void keyPressed() {
	if (key == ‘ ‘) {
	client.stop();
	} else if (key == ‘d‘) {
	println(client.description());
	}
	}
	public void found(int i) {
	//println(“found: ” + i);
	found = i;
	}
	public void switchImg() {
	//println(“found: ” + i);
	}
	public void rawRecieved(float[] f) {
	// println(“mouth: ” + f[0]);
	//pointx = f[0];
	//pointy = f[1];
	for (int i=0; i < f.length; i++){
	rawPoints[i] = f[i];
	}
	//nostrils = f;
	}
	// all other OSC messages end up here
	void oscEvent(OscMessage m) {
	if(m.isPlugged() == false) {
	//println(“UNPLUGGED: ” + m);
	}
	}

 

Here’s an example of one of my trials:

https://vimeo.com/157921636

and another: https://vimeo.com/157921629

Here is a video documenting how it all works:

Here is the code (with the lyrics I use in the video commented):

import oscP5.*;
import netP5.*;

// make osc object
OscP5 oscP5;

// variables that store received mouth width and height
float mouthWidth = 0;
float mouthHeight = 0;
int counter = 0;
int mouth = 0;
int test =0;

float xPos;
float yPos;

String[] words = //{“Happiness hit her like a train on a track”, “Coming towards her stuck still no turning back”,
//”She hid around corners and she hid under beds”,
//”She killed it with kisses and from it she fled”,
//”With every bubble she sank with her drink”,
//”And washed it away down the kitchen sink”,
//”The dog days are over”,
//”The dog days are done”,
//”The horses are coming”,
//”So you better run”,
//”Run fast for your mother, run fast for your father”,
//”Run for your children, for your sisters and brothers”,
//”Leave all your love and your longing behind”,
//”You can’t carry it with you if you want to survive”,
//};
//{“hello”, “it’s me”, “i was wondering if after all these years you’d like to meet”, “to go over”, “everything”, “They say that time’s supposed to heal ya”,
//”But I ain’t done much healing”,”Hello, can you hear me?”, “I’m in California dreaming about who we used to be”, “When we were younger and free”,
//”I’ve forgotten how it felt before the world fell at our feet”, “There’s such a difference between us”, “And a million miles”,
//”Hello from the other side”,

{
“A moment of love”,
“A dream”,
“A laugh”,
“A kiss”,
“A cry”,
“Our rights”,
“Our wrongs”,
“A moment of love”,
“A dream”,
“A laugh”,
“A moment of love”,
“A dream”,
“A laugh”,
“So stay there”,
“‘Cause I’ll be coming over”,
“While our blood’s still young”,
“It’s so young, it runs”,
“Won’t stop ’til it’s over”,
“Won’t stop to surrender”,
};
void setup() {
background(90);
size(640, 480);

// note: port needs to match FaceOSC port
oscP5 = new OscP5(this, 8338);

// get face position
oscP5.plug(this, “getPosition”, “/pose/position”);
// set up the function that will process the incoming data
// with the osc address “/gesture/mouth/width”
oscP5.plug(this, “gotMouthWidth”, “/gesture/mouth/width”);

// same for height
oscP5.plug(this, “gotMouthHeight”, “/gesture/mouth/height”);
}
void draw() {
background(250);
println(“Counter: ” + counter);
if(mouthHeight > 2){
mouth = 1;
}
if(mouth == 1){
mouth = 0;
if(counter < words.length){
fill(0);
text(words[counter], xPos, yPos);
test = 1;
}
}
else{
mouth = 0;
if(test == 1){
counter ++;
test = 0;
}
background(123,34,189);
}

}

public void gotMouthWidth( float mouthW ) {
//println(“mouthWidth: ” + mouthWidth);
mouthWidth = mouthW;
}

public void gotMouthHeight( float mouthH ) {
//println(“mouthHeight: ” + mouthH);
mouthHeight = mouthH;
}

public void getPosition(float x, float y) {
//println(“get position\tX: ” + x + ” Y: ” + y );
xPos = x;
yPos = y;
}