JMF

• JMF is a set of classes to handle multimedia data
• It can deal with audio and video
• It can read from sources and write to sinks
• Sources can be files, devices or the network
• Sinks can be files, devices or the network

Audio file formats

There are a huge number of different file formats

• PCM (Pulse code modulated): raw digital data e.g. from a CD
• ALAW: U.S. and international standards for logarithmic telephone sound compression
• GSM: a standard used for telephone sound compression in European countries
• Music sound formats such as AU, WAV, MP3, ...

Video formats

There are also a huge number of video formats

• MPEG
• AVI
• Quicktime
• WMF

Handling formats

• JMF does its best to handle all these formats with as little programmer intervention as possible
• It detects devices automatically, and uses converters automatically
• It doesn't seem to leave enough programmer control at times

Player

• JMF uses a Player to deliver data to a suitable output device
• The output device is selected by the Player based on media type - if possible
• Players are created using a Manager factory
• This allows the set of Players to be extended
• A Player will generate events such as StartEvent, StopEvent
• A Player has methods such as stop() and start() plus many others for control of media and time

Play a file

import java.io.*;

import javax.media.*;
import javax.media.format.*;
import javax.media.protocol.*;

public class PlayFile implements ControllerListener {
    // private String srcFile = "src.au";
    // private String srcFile = "tull.mpg";
    private String srcFile = "the_privilege.mpg";

    public static void main(String [] args) {
	new PlayFile();
    }

    public PlayFile() {
	Player p;

	MediaLocator src = new MediaLocator("file:" + srcFile);

	try {
	    p = Manager.createPlayer(src);
	    p.addControllerListener(this);
	    p.start();
	} catch(Exception e) {
	    e.printStackTrace();
	    System.exit(1);
	}
    }

    public synchronized void controllerUpdate(ControllerEvent evt) {
	if (evt instanceof EndOfMediaEvent) {
	    System.out.println("Play finished");
	    System.exit(0);
	} else {
	    System.out.println(evt.toString());
	}
    }
}

Presentation

• A Player can be controlled by calling its methods
• A Player may also have an associated user interface for controlling it
• The control panel is obtained by getControlPanelComponent() which returns an AWT Component
• The control panel may have e.g. buttons for stop and start
• A Player which can display video data can also have a visual component which is obtained by getVisualComponent()

Play a file with a UI

import java.io.*;
import java.awt.*;

import javax.media.*;
import javax.media.format.*;
import javax.media.protocol.*;

public class PlayFileGUI extends Frame implements ControllerListener {
    private boolean realized = false;
    private boolean configured = false;
    private Player p = null;

    public static void main(String [] args) {
	new PlayFileGUI();
    }

    public PlayFileGUI() {
	MediaLocator src = new MediaLocator("file:" + "src.wav");

	try {
	    p = Manager.createPlayer(src);
	    p.addControllerListener(this);
	    p.realize();
	} catch(Exception e) {
	    e.printStackTrace();
	    System.exit(1);
	}
    }

    public synchronized void controllerUpdate(ControllerEvent evt) {
	if (evt instanceof RealizeCompleteEvent) {
	    Component controlPanel = p.getControlPanelComponent();
	    add(controlPanel, BorderLayout.CENTER);
	    pack();
	    setVisible(true);
	} else {
	    // System.out.println(evt.toString());
	}
    }
}

Processor

• To manipulate data a Processor object is used
• A Processor has a source and a sink
• Based on the media types of the source and the sink, the Presentation may convert the data from one format to another
• A Processor goes through the stages of
  • configuration
  • realization
  • prefetching data
  • started
• Selection of a converter is supposed to take place automatically, but may not
  • When a processor is created (from a Manager) the source is set
  • The sink cannot be set until the processor has been realized
  • But, the processor may do things after configuration but before realization that limit possible sink types
  • Note: configure() and realize() act asynchronously, and return immediately

Copy a file

import java.io.*;
import java.util.Vector;

import javax.media.*;
import javax.media.format.*;
import javax.media.protocol.*;

public class Copy implements ControllerListener {
    private boolean realized = false;
    private boolean configured = false;

    public static void main(String [] args) {
	new Copy();
    }

    public Copy() {
	Processor p;

	DataSink sink;
	MediaLocator src = new MediaLocator("file:" + "tull.mp3");
	MediaLocator dest = new MediaLocator("file:" + "tmp.mp3");

	try {
	    p = Manager.createProcessor(src);
	    p.addControllerListener(this);

	    p.configure();
	    while (! configured) {
		try {
		    Thread.currentThread().sleep(100L);;
		} catch (InterruptedException e) {
		    // ignore
		}
	    }

	    ContentDescriptor[] descriptors = p.getSupportedContentDescriptors();
	    for (int n = 0; n < descriptors.length; n++) {
		System.out.println("Desc: " + descriptors[n].toString());
	    }
	    p.setContentDescriptor(new ContentDescriptor(FileTypeDescriptor.MPEG_AUDIO));

	    p.realize();
	    while (! realized) {
		try {
		    Thread.currentThread().sleep(100L);;
		} catch (InterruptedException e) {
		    // ignore
		}
	    }

	    DataSource output = p.getDataOutput();

	    System.out.println("DataSource type: ");
	    Class cls = output.getClass();
	    while (cls != null) {
		System.out.println(cls.toString());
		cls = cls.getSuperclass();
	    }

	    sink = Manager.createDataSink(output, dest);
	    sink.open();
	    sink.start();
	    p.start();

	} catch(Exception e) {
	    e.printStackTrace();
	    System.exit(1);
	}
    }

    public synchronized void controllerUpdate(ControllerEvent evt) {
	if (evt instanceof RealizeCompleteEvent) {
	    realized = true;
	} else 	if (evt instanceof ConfigureCompleteEvent) {
	    configured = true;
	} else if (evt instanceof EndOfMediaEvent) {
	    System.exit(0);
	} else {
	    // System.out.println(evt.toString());
	}
    }
}

Capture devices

• When JMF is installed, it attempts to find all the A/V devices attached to the computer
• This list is stored in the JMF properties files, and is used to determine which DLLs are loaded
• A CaptureDeviceManager can be queried for all devices, or for devices supporting a particular Format e.g. an AudioFormat supporting a LINEAR encoding with a sample rate of 44100 samples per second, 16 bits stereo (a CD player)

List capture devices

import java.io.*;
import java.util.Vector;

import javax.media.*;
import javax.media.format.*;


public class Devices {

    public static void main(String [] args) {

	Vector devices= CaptureDeviceManager.getDeviceList(null);
	System.out.println("Number of capture devices: " + devices.size());
	for (int n = 0; n < devices.size(); n++) {
	    CaptureDeviceInfo info = (CaptureDeviceInfo) devices.elementAt(n);
	    System.out.println(info.toString());
	}
    }
}

Playing from a device

This program should copy from the sound card to a file. It doesn't work on my laptop as "interface TargetDataLine supporting format PCM_SIGNED..." is not found

import java.io.*;
import java.util.Vector;

import javax.media.*;
import javax.media.format.*;
import javax.media.protocol.*;

public class SoundCardToFile implements ControllerListener {

    public static void main(String [] args) {
	new SoundCardToFile();
    }

    public SoundCardToFile() {
	Processor p = null;

	AudioFormat format= new AudioFormat(AudioFormat.LINEAR); //, 
	// 44100, 
	//			    16, 
	//			    2); 

	try {
	    p = Manager.createRealizedProcessor(
                        new ProcessorModel(new Format[] {format},
			    new ContentDescriptor(ContentDescriptor.RAW)));
	} catch (Exception e) {
	    System.err.println(e.toString());
	    System.exit(1);
	}
	System.out.println("GOT HERE");

	DataSink sink;
	MediaLocator dest = new MediaLocator("file:" + "copy.au");

	try {
	    DataSource output = p.getDataOutput();
	    sink = Manager.createDataSink(output, dest);
	    sink.open();
	    sink.start();
	    p.start();
	   
	} catch(Exception e) {
	    e.printStackTrace();
	    System.exit(1);
	}
    }

    public synchronized void controllerUpdate(ControllerEvent evt) {
	if (evt instanceof EndOfMediaEvent) {
	    System.exit(0);
	} else {
	    // System.out.println(evt.toString());
	}
    }
}

Real-time protocol RTP

• LANs based on ethernet are not "real time" with guaranteed time of delivery of packets, due to the collision algorithm used (back off a random amount of time)
• The IP protocol does not guarantee delivery between hosts, let alone a guaranteed time of delivery
• TCP gives reliable delivery between ports, or at least a failure message
• UDP gives unreliable delivery between ports, with no time guarantee. But in practise, on LANs, UDP is reliable and time is not usually too much of a problem
• RTP sits above UDP, and offers limited guarantees of delivery and timeliness

RTP client

The following client uses a simple way of getting streaming RTP data from a server. It must be running on a different machine to the server. If you try on the same machine, then since they both use UDP they will try to connect to the same port and one will fail.

import java.io.*;

import javax.media.*;
import javax.media.format.*;
import javax.media.protocol.*;

public class RTPClient implements ControllerListener {

    public static void main(String [] args) {
	new RTPClient();
    }

    public RTPClient() {
	Player p;

	String srcUrl = "rtp://192.168.2.1:42050/audio/1";
	DataSink sink;
	MediaLocator src = new MediaLocator(srcUrl);

	try {
	    p = Manager.createPlayer(src);
	    p.addControllerListener(this);
	    p.start();
	} catch(Exception e) {
	    e.printStackTrace();
	    System.exit(1);
	}
    }

    public synchronized void controllerUpdate(ControllerEvent evt) {
	if (evt instanceof EndOfMediaEvent) {
	    System.exit(0);
	} else {
	    System.out.println(evt.toString());
	}
    }
}

RTP server

This must tbe run on a different machine to the client.

import java.io.*;
import java.util.Vector;
import java.net.InetAddress;
import javax.media.rtp.*;
import javax.media.rtp.rtcp.*;
import javax.media.*;
import javax.media.format.*;
import javax.media.protocol.*;
import javax.media.control.*;

public class RTPServer implements ControllerListener {
    private boolean realized = false;
    private boolean configured = false;
    private String ipAddress = "192.168.2.11";
    private String destUrl = "rtp://192.168.2.11:42050/audio/1";
    private MediaLocator dest = new MediaLocator(destUrl);

    public static void main(String [] args) {
	new RTPServer();
    }

    public RTPServer() {
	Processor p;

	String srcFile = "long.wav";
	MediaLocator src = new MediaLocator("file:" + srcFile);

	try {
	    p = Manager.createProcessor(src);
	    p.addControllerListener(this);

	    p.configure();
	    while (! configured) {
		try {
		    Thread.currentThread().sleep(100L);;
		} catch (InterruptedException e) {
		    // ignore
		}
	    }

	    setTrackFormat(p);
	    p.setContentDescriptor(new ContentDescriptor(ContentDescriptor.RAW_RTP));

	    p.realize();
	    while (! realized) {
		try {
		    Thread.currentThread().sleep(100L);;
		} catch (InterruptedException e) {
		    // ignore
		}
	    }

	    transmit(p);

	} catch(Exception e) {
	    e.printStackTrace();
	    System.exit(1);
	}
    }


    /**
     * It doesn't seem to be enough to set the processor output format.
     * This code sets the format for each track within the stream to a 
     * supported format. On my system this a 4-bit format which sounds
     * crappy, but is recognised by the client. 
     * Without this track setting, the client barfs because the default
     * track format is a 16-bit mpeg stream which the client doesn't
     * understand
     */
    private void setTrackFormat(Processor p) {
	// Get the tracks from the processor
	TrackControl [] tracks = p.getTrackControls();
	
	// Do we have atleast one track?
	if (tracks == null || tracks.length < 1) {
	    System.out.println("Couldn't find tracks in processor");
	    System.exit(1);
	}
	
	// Set the output content descriptor to RAW_RTP
	// This will limit the supported formats reported from
	// Track.getSupportedFormats to only valid RTP formats.
	ContentDescriptor cd = new ContentDescriptor(ContentDescriptor.RAW_RTP);
	p.setContentDescriptor(cd);
	
	Format supported[];
	Format chosen;
	boolean atLeastOneTrack = false;
	
	// Program the tracks.
	for (int i = 0; i < tracks.length; i++) {
	    Format format = tracks[i].getFormat();
	    if (tracks[i].isEnabled()) {
		
		supported = tracks[i].getSupportedFormats();
		for (int n = 0; n < supported.length; n++)
		    System.out.println("Supported format: " + supported[n]);
		
		// We've set the output content to the RAW_RTP.
		// So all the supported formats should work with RTP.
		// We'll just pick the first one.
		
		if (supported.length > 0) {
		    chosen = supported[0];
		    tracks[i].setFormat(chosen);
		    System.err.println("Track " + i + " is set to transmit as:");
		    System.err.println("  " + chosen);
		    atLeastOneTrack = true;
		} else
		    tracks[i].setEnabled(false);
	    } else
		tracks[i].setEnabled(false);
	}
    }

    /**
     * This code uses the RTP Manager to handle a session
     * stream. It is more complex than the simpler version
     * below, but can also handle more complex streams

    private void transmit(Processor p) {
	try {
	    DataSource output = p.getDataOutput();
	    PushBufferDataSource pbds = (PushBufferDataSource) output;
	    RTPManager rtpMgr = RTPManager.newInstance();
	    SessionAddress localAddr, destAddr;
	    SendStream sendStream;
	    int port = 42506;
	    SourceDescription srcDesList[];

	    localAddr = new SessionAddress( InetAddress.getLocalHost(),
					    port);
	    
	    InetAddress	ipAddr = InetAddress.getByName(ipAddress);
	    destAddr = new SessionAddress( ipAddr, port);
	    
	    rtpMgr.initialize(localAddr);
	    
	    rtpMgr.addTarget(destAddr);
	    
	    System.err.println( "Created RTP session: " + ipAddress + " " + port);
	    
	    sendStream = rtpMgr.createSendStream(output, 0);		
	    sendStream.start();

	    p.start();
	} catch(Exception e) {
	    e.printStackTrace();
	}
    }
    */

    private void transmit(Processor p) {
	try {
	    DataSource output = p.getDataOutput();
	    DataSink rtpSink;
	    rtpSink = Manager.createDataSink(output, dest);
	    System.out.println("Sink content type: " + rtpSink.getContentType());
	    System.out.println("Sink media type: " + rtpSink.getOutputLocator().toString());
	    rtpSink.open();
	    rtpSink.start();
	    p.start();
	} catch(Exception e) {
	    e.printStackTrace();
	}
    }

    public synchronized void controllerUpdate(ControllerEvent evt) {
	if (evt instanceof RealizeCompleteEvent) {
	    realized = true;
	} else 	if (evt instanceof ConfigureCompleteEvent) {
	    configured = true;
	} else if (evt instanceof EndOfMediaEvent) {
	    System.exit(0);
	} else {
	    // System.out.println(evt.toString());
	}
    }
}